Implement Node.js Streams support for `chainStreams` #64948

Ethan-Arrowood · 2024-04-23T22:01:53Z

Implements support for Node.js Streams in the chainStreams method.

Ethan-Arrowood · 2024-04-23T22:02:08Z

Warning

This pull request is not mergeable via GitHub because a downstack PR is open. Once all requirements are satisfied, merge this PR as a stack on Graphite.
Learn more

Implement Node.js Streams support for chainStreams #64948 👈
Experimental Node.js Streams support #64932
canary

This stack of pull requests is managed by Graphite. Learn more about stacking.

Join @Ethan-Arrowood and the rest of your teammates on Graphite

ijjk · 2024-04-23T22:04:59Z

Failing test suites

Commit: 7edeac4

TURBOPACK=1 pnpm test test/integration/amphtml-ssg/test/index.test.js (turbopack)

AMP SSG Support > export mode > production mode > should have copied SSG files correctly

Expand output

● AMP SSG Support › export mode › production mode › should have copied SSG files correctly

command failed with code 1 signal null
  ▲ Next.js 14.3.0-canary.21 (turbo)

   Linting and checking validity of types ...

 ⚠ The Next.js plugin was not detected in your ESLint configuration. See https://nextjs.org/docs/basic-features/eslint#migrating-existing-config
   Creating an optimized production build ...
next-swc build: local built @next/swc from NEXT_TEST_NATIVE_DIR
   Building (0/5) ...
   Building (1/5) 
   Building (2/5) 
   Building (3/5) 
 ✓ Building (5/5)
   Collecting page data ...
   Generating static pages (0/7) ...
   Generating static pages (1/7) 
   Generating static pages (3/7) 

Error occurred prerendering page "/blog/post-2". Read more: https://nextjs.org/docs/messages/prerender-error

Error: Could not read from https://cdn.ampproject.org/v0/validator_wasm.js - aborted

  15 |  * See the License for the specific language governing permissions and
  16 |  * limitations under the license.
> 17 |  */const t=__nccwpck_require__(915);const r=__nccwpck_require__(147);const i=__nccwpck_require__(685);const s=__nccwpck_require__(687);const o=__nccwpck_require__(17);const a=__nccwpck_require__(642);const l=__nccwpck_require__(263);const u=__nccwpck_require__(477);const c=__nccwpck_require__(310);const h=__nccwpck_require__(837);const p=__nccwpck_require__(144);const f="amphtml-validator";function hasPrefix(e,t){return e.indexOf(t)==0}function isHttpOrHttpsUrl(e){return hasPrefix(e,"http://")||hasPrefix(e,"https://")}function readFromFile(e){return new l((function(t,n){r.readFile(e,"utf8",(function(e,r){if(e){n(e)}else{t(r.trim())}}))}))}function readFromReadable(e,t){return new l((function(n,r){const i=[];t.setEncoding("utf8");t.on("data",(function(e){i.push(e)}));t.on("end",(function(){n(i.join(""))}));t.on("error",(function(t){r(new Error("Could not read from "+e+" - "+t.message))}))}))}function readFromStdin(){return readFromReadable("stdin",process.stdin).then((function(e){process.stdin.resume();return e}))}function readFromUrl(e,t){return new l((function(n,r){const o=hasPrefix(e,"http://")?i:s;const a=o.request(e,(function(t){if(t.statusCode!==200){t.resume();r(new Error("Unable to fetch "+e+" - HTTP Status "+t.statusCode))}else{n(t)}}));a.setHeader("User-Agent",t);a.on("error",(function(t){r(new Error("Unable to fetch "+e+" - "+t.message))}));a.end()})).then(readFromReadable.bind(null,e))}function ValidationResult(){this.status="UNKNOWN";this.errors=[]}function ValidationError(){this.severity="UNKNOWN_SEVERITY";this.line=1;this.col=0;this.message="";this.specUrl=null;this.code="UNKNOWN_CODE";this.params=[]}function Validator(e){this.sandbox=p.createContext();try{new p.Script(e).runInContext(this.sandbox)}catch(e){throw new Error("Could not instantiate validator_wasm.js - "+e.message)}}Validator.prototype.init=function(){if(this.sandbox.amp.validator.init){return this.sandbox.amp.validator.init()}else{return l.resolve(undefined)}};Validator.prototype.validateString=function(e,t){const n=this.sandbox.amp.validator.validateString(e,t);const r=new ValidationResult;r.status=n.status;for(let e=0;e<n.errors.length;e++){const t=n.errors[e];const i=new ValidationError;i.severity=t.severity;i.line=t.line;i.col=t.col;i.message=this.sandbox.amp.validator.renderErrorMessage(t);i.specUrl=t.specUrl;i.code=t.code;i.params=t.params;r.errors.push(i)}return r};const m={};function getInstance(e,t){const n=e||"https://cdn.ampproject.org/v0/validator_wasm.js";const r=t||f;if(m.hasOwnProperty(n)){return l.resolve(m[n])}const i=isHttpOrHttpsUrl(n)?readFromUrl(n,r):readFromFile(n);return i.then((function(e){let t;try{t=new Validator(e)}catch(e){throw e}m[n]=t;return t})).then((function(e){return e.init().then((()=>e))}))}e.getInstance=getInstance;function newInstance(e){return new Validator(e)}e.newInstance=newInstance;function logValidationResult(e,n,r){if(n.status==="PASS"){process.stdout.write(e+": "+(r?t.green("PASS"):"PASS")+"\n")}for(let i=0;i<n.errors.length;i++){const s=n.errors[i];let o=e+":"+s.line+":"+s.col+" ";if(r){o+=(s.severity==="ERROR"?t.red:t.magenta)(s.message)}else{o+=s.message}if(s.specUrl){o+=" (see "+s.specUrl+")"}process.stderr.write(o+"\n")}}function main(){a.usage("[options] <fileOrUrlOrMinus...>\n\n"+'  Validates the files or urls provided as arguments. If "-" is\n'+"  specified, reads from stdin instead.").option("--validator_js <fileOrUrl>","The Validator Javascript.\n"+"  Latest published version by default, or\n"+"  dist/validator_minified.js (built with build.py)\n"+"  for development.","https://cdn.ampproject.org/v0/validator_wasm.js").option("--user-agent <userAgent>","User agent string to use in requests.",f).option("--html_format <AMP|AMP4ADS|AMP4EMAIL>","The input format to be validated.\n"+"  AMP by default.","AMP").option("--format <color|text|json>","How to format the output.\n"+'  "color" displays errors/warnings/success in\n'+"          red/orange/green.\n"+'  "text"  avoids color (e.g., useful in terminals not\n'+"          supporting color).\n"+'  "json"  emits json corresponding to the ValidationResult\n'+"          message in validator.proto.","color").parse(process.argv);const e=a.opts();if(e.length===0){a.outputHelp();process.exit(1)}if(e.html_format!=="AMP"&&e.html_format!=="AMP4ADS"&&e.html_format!=="AMP4EMAIL"){process.stderr.write('--html_format must be set to "AMP", "AMP4ADS", or "AMP4EMAIL".\n',(function(){process.exit(1)}))}if(e.format!=="color"&&e.format!=="text"&&e.format!=="json"){process.stderr.write('--format must be set to "color", "text", or "json".\n',(function(){process.exit(1)}))}const n=[];for(let t=0;t<a.args.length;t++){const r=a.args[t];if(r==="-"){n.push(readFromStdin())}else if(isHttpOrHttpsUrl(r)){n.push(readFromUrl(r,e.userAgent))}else{n.push(readFromFile(r))}}getInstance(e.validator_js,e.userAgent).then((function(r){l.all(n).then((function(t){const n={};let i=false;for(let s=0;s<t.length;s++){const o=r.validateString(t[s],e.html_format);if(e.format==="json"){n[a.args[s]]=o}else{logValidationResult(a.args[s],o,e.format==="color"?true:false)}if(o.status!=="PASS"){i=true}}if(e.format==="json"){process.stdout.write(JSON.stringify(n)+"\n",(function(){process.exit(i?1:0)}))}else if(i){process.stderr.write("",(function(){process.exit(1)}))}else{process.stdout.write("",(function(){process.exit(0)}))}})).catch((function(n){process.stderr.write((e.format=="color"?t.red(n.message):n.message)+"\n",(function(){process.exit(1)}))}))})).catch((function(n){process.stderr.write((e.format=="color"?t.red(n.message):n.message)+"\n",(function(){process.exit(1)}))}))}e.main=main})();module.exports=n})();
     |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              ^

  at IncomingMessage.<anonymous> (../packages/next/dist/compiled/amphtml-validator/index.js:17:846)
     Generating static pages (5/7) 
   ✓ Generating static pages (7/7)
  > Export encountered errors on following paths:
  	/blog/[slug]: /blog/post-2
  at ChildProcess.<anonymous> (lib/next-test-utils.ts:284:11)

Read more about building and testing Next.js in contributing.md.

pnpm test test/integration/prerender-export/test/index.test.js

Expand output

● Test suite failed to run

thrown: "Exceeded timeout of 60000 ms for a hook.
Add a timeout value to this test to increase the timeout, if this is a long-running test. See https://jestjs.io/docs/api#testname-fn-timeout."

  219 |         })
  220 |
> 221 |         afterAll(async () => {
      |         ^
  222 |           if (app) {
  223 |             await stopApp(app)
  224 |           }

  at afterAll (integration/prerender-export/test/index.test.js:221:9)
  at describe (integration/prerender-export/test/index.test.js:211:7)
  at integration/prerender-export/test/index.test.js:208:56
  at Object.describe (integration/prerender-export/test/index.test.js:207:1)

Read more about building and testing Next.js in contributing.md.

ijjk · 2024-04-24T17:27:17Z

Stats from current PR

Default Build (Increase detected ⚠️)

General Overall increase ⚠️

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
buildDuration	17.6s	16.2s	N/A
buildDurationCached	8.6s	7.3s	N/A
nodeModulesSize	238 MB	239 MB	⚠️ +518 kB
nextStartRea..uration (ms)	392ms	413ms	N/A

Client Bundles (main, webpack)

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
2453-HASH.js gzip	31.5 kB	31.5 kB	N/A
3304.HASH.js gzip	169 B	169 B	✓
3f784ff6-HASH.js gzip	53.7 kB	53.7 kB	N/A
8299-HASH.js gzip	5.09 kB	5.09 kB	✓
framework-HASH.js gzip	45.2 kB	45.2 kB	✓
main-app-HASH.js gzip	229 B	227 B	N/A
main-HASH.js gzip	31.5 kB	31.6 kB	N/A
webpack-HASH.js gzip	1.64 kB	1.65 kB	N/A
Overall change	50.5 kB	50.5 kB	✓

Legacy Client Bundles (polyfills)

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
polyfills-HASH.js gzip	31 kB	31 kB	✓
Overall change	31 kB	31 kB	✓

Client Pages

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
_app-HASH.js gzip	193 B	194 B	N/A
_error-HASH.js gzip	193 B	191 B	N/A
amp-HASH.js gzip	510 B	510 B	✓
css-HASH.js gzip	342 B	343 B	N/A
dynamic-HASH.js gzip	2.51 kB	2.52 kB	N/A
edge-ssr-HASH.js gzip	265 B	265 B	✓
head-HASH.js gzip	365 B	364 B	N/A
hooks-HASH.js gzip	389 B	391 B	N/A
image-HASH.js gzip	4.28 kB	4.28 kB	N/A
index-HASH.js gzip	269 B	268 B	N/A
link-HASH.js gzip	2.68 kB	2.69 kB	N/A
routerDirect..HASH.js gzip	328 B	326 B	N/A
script-HASH.js gzip	395 B	397 B	N/A
withRouter-HASH.js gzip	323 B	323 B	✓
1afbb74e6ecf..834.css gzip	106 B	106 B	✓
Overall change	1.2 kB	1.2 kB	✓

Client Build Manifests

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
_buildManifest.js gzip	482 B	484 B	N/A
Overall change	0 B	0 B	✓

Rendered Page Sizes

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
index.html gzip	529 B	528 B	N/A
link.html gzip	541 B	541 B	✓
withRouter.html gzip	524 B	523 B	N/A
Overall change	541 B	541 B	✓

Edge SSR bundle Size Overall increase ⚠️

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
edge-ssr.js gzip	108 kB	108 kB	⚠️ +366 B
page.js gzip	3.05 kB	3.04 kB	N/A
Overall change	108 kB	108 kB	⚠️ +366 B

Middleware size

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
middleware-b..fest.js gzip	623 B	623 B	✓
middleware-r..fest.js gzip	155 B	156 B	N/A
middleware.js gzip	27.8 kB	27.8 kB	N/A
edge-runtime..pack.js gzip	839 B	839 B	✓
Overall change	1.46 kB	1.46 kB	✓

Next Runtimes Overall increase ⚠️

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
app-page-exp...dev.js gzip	171 kB	172 kB	⚠️ +1.16 kB
app-page-exp..prod.js gzip	97.6 kB	98.9 kB	⚠️ +1.3 kB
app-page-tur..prod.js gzip	99.4 kB	101 kB	⚠️ +1.3 kB
app-page-tur..prod.js gzip	93.6 kB	95 kB	⚠️ +1.35 kB
app-page.run...dev.js gzip	145 kB	146 kB	⚠️ +1.23 kB
app-page.run..prod.js gzip	92.1 kB	93.5 kB	⚠️ +1.34 kB
app-route-ex...dev.js gzip	21.5 kB	21.5 kB	✓
app-route-ex..prod.js gzip	15.1 kB	15.1 kB	✓
app-route-tu..prod.js gzip	15.1 kB	15.1 kB	✓
app-route-tu..prod.js gzip	14.9 kB	14.9 kB	✓
app-route.ru...dev.js gzip	21.2 kB	21.2 kB	✓
app-route.ru..prod.js gzip	14.9 kB	14.9 kB	✓
pages-api-tu..prod.js gzip	9.55 kB	9.55 kB	✓
pages-api.ru...dev.js gzip	9.82 kB	9.82 kB	✓
pages-api.ru..prod.js gzip	9.55 kB	9.55 kB	✓
pages-turbo...prod.js gzip	21.4 kB	23.7 kB	⚠️ +2.31 kB
pages.runtim...dev.js gzip	22.1 kB	24.4 kB	⚠️ +2.34 kB
pages.runtim..prod.js gzip	21.4 kB	23.7 kB	⚠️ +2.31 kB
server.runti..prod.js gzip	65.3 kB	67.7 kB	⚠️ +2.34 kB
Overall change	960 kB	977 kB	⚠️ +17 kB

build cache

	vercel/next.js canary	vercel/next.js 04-23-implement_chainStreams	Change
0.pack gzip	1.61 MB	1.61 MB	N/A
index.pack gzip	107 kB	107 kB	N/A
Overall change	0 B	0 B	✓

Diff details

Diff for middleware.js

Diff too large to display

Diff for edge-ssr.js

Diff too large to display

Diff for image-HASH.js

@@ -1,7 +1,7 @@
 (self["webpackChunk_N_E"] = self["webpackChunk_N_E"] || []).push([
   [8358],
   {
-    /***/ 1552: /***/ (
+    /***/ 4070: /***/ (
       __unused_webpack_module,
       __unused_webpack_exports,
       __webpack_require__
@@ -9,7 +9,7 @@
       (window.__NEXT_P = window.__NEXT_P || []).push([
         "/image",
         function () {
-          return __webpack_require__(5237);
+          return __webpack_require__(396);
         },
       ]);
       if (false) {
@@ -18,7 +18,7 @@
       /***/
     },
 
-    /***/ 2016: /***/ (module, exports, __webpack_require__) => {
+    /***/ 8490: /***/ (module, exports, __webpack_require__) => {
       "use strict";
       /* __next_internal_client_entry_do_not_use__  cjs */
       Object.defineProperty(exports, "__esModule", {
@@ -40,15 +40,15 @@
         __webpack_require__(422)
       );
       const _head = /*#__PURE__*/ _interop_require_default._(
-        __webpack_require__(6074)
+        __webpack_require__(2457)
       );
-      const _getimgprops = __webpack_require__(9571);
-      const _imageconfig = __webpack_require__(6567);
-      const _imageconfigcontextsharedruntime = __webpack_require__(419);
-      const _warnonce = __webpack_require__(4486);
-      const _routercontextsharedruntime = __webpack_require__(162);
+      const _getimgprops = __webpack_require__(7932);
+      const _imageconfig = __webpack_require__(5706);
+      const _imageconfigcontextsharedruntime = __webpack_require__(9483);
+      const _warnonce = __webpack_require__(9035);
+      const _routercontextsharedruntime = __webpack_require__(4829);
       const _imageloader = /*#__PURE__*/ _interop_require_default._(
-        __webpack_require__(6996)
+        __webpack_require__(7240)
       );
       // This is replaced by webpack define plugin
       const configEnv = {
@@ -379,7 +379,7 @@
       /***/
     },
 
-    /***/ 9571: /***/ (
+    /***/ 7932: /***/ (
       __unused_webpack_module,
       exports,
       __webpack_require__
@@ -395,9 +395,9 @@
           return getImgProps;
         },
       });
-      const _warnonce = __webpack_require__(4486);
-      const _imageblursvg = __webpack_require__(133);
-      const _imageconfig = __webpack_require__(6567);
+      const _warnonce = __webpack_require__(9035);
+      const _imageblursvg = __webpack_require__(2642);
+      const _imageconfig = __webpack_require__(5706);
       const VALID_LOADING_VALUES =
         /* unused pure expression or super */ null && [
           "lazy",
@@ -772,7 +772,7 @@
       /***/
     },
 
-    /***/ 133: /***/ (__unused_webpack_module, exports) => {
+    /***/ 2642: /***/ (__unused_webpack_module, exports) => {
       "use strict";
       /**
        * A shared function, used on both client and server, to generate a SVG blur placeholder.
@@ -827,7 +827,7 @@
       /***/
     },
 
-    /***/ 4085: /***/ (
+    /***/ 503: /***/ (
       __unused_webpack_module,
       exports,
       __webpack_require__
@@ -854,10 +854,10 @@
         },
       });
       const _interop_require_default = __webpack_require__(7456);
-      const _getimgprops = __webpack_require__(9571);
-      const _imagecomponent = __webpack_require__(2016);
+      const _getimgprops = __webpack_require__(7932);
+      const _imagecomponent = __webpack_require__(8490);
       const _imageloader = /*#__PURE__*/ _interop_require_default._(
-        __webpack_require__(6996)
+        __webpack_require__(7240)
       );
       function getImageProps(imgProps) {
         const { props } = (0, _getimgprops.getImgProps)(imgProps, {
@@ -889,7 +889,7 @@
       /***/
     },
 
-    /***/ 6996: /***/ (__unused_webpack_module, exports) => {
+    /***/ 7240: /***/ (__unused_webpack_module, exports) => {
       "use strict";
 
       Object.defineProperty(exports, "__esModule", {
@@ -924,7 +924,7 @@
       /***/
     },
 
-    /***/ 5237: /***/ (
+    /***/ 396: /***/ (
       __unused_webpack_module,
       __webpack_exports__,
       __webpack_require__
@@ -941,8 +941,8 @@
 
       // EXTERNAL MODULE: ./node_modules/.pnpm/react@18.2.0/node_modules/react/jsx-runtime.js
       var jsx_runtime = __webpack_require__(1527);
-      // EXTERNAL MODULE: ./node_modules/.pnpm/file+..+main-repo+packages+next+next-packed.tgz_react-dom@18.2.0_react@18.2.0/node_modules/next/image.js
-      var next_image = __webpack_require__(1577);
+      // EXTERNAL MODULE: ./node_modules/.pnpm/file+..+diff-repo+packages+next+next-packed.tgz_react-dom@18.2.0_react@18.2.0/node_modules/next/image.js
+      var next_image = __webpack_require__(73);
       var image_default = /*#__PURE__*/ __webpack_require__.n(next_image); // CONCATENATED MODULE: ./pages/nextjs.png
       /* harmony default export */ const nextjs = {
         src: "/_next/static/media/nextjs.cae0b805.png",
@@ -972,12 +972,8 @@
       /***/
     },
 
-    /***/ 1577: /***/ (
-      module,
-      __unused_webpack_exports,
-      __webpack_require__
-    ) => {
-      module.exports = __webpack_require__(4085);
+    /***/ 73: /***/ (module, __unused_webpack_exports, __webpack_require__) => {
+      module.exports = __webpack_require__(503);
 
       /***/
     },
@@ -987,7 +983,7 @@
     /******/ var __webpack_exec__ = (moduleId) =>
       __webpack_require__((__webpack_require__.s = moduleId));
     /******/ __webpack_require__.O(0, [2888, 9774, 179], () =>
-      __webpack_exec__(1552)
+      __webpack_exec__(4070)
     );
     /******/ var __webpack_exports__ = __webpack_require__.O();
     /******/ _N_E = __webpack_exports__;

Diff for 2453-HASH.js

Diff too large to display

Diff for main-HASH.js

Diff too large to display

Diff for app-page-exp..ntime.dev.js

failed to diff

Diff for app-page-exp..time.prod.js

Diff too large to display

Diff for app-page-tur..time.prod.js

Diff too large to display

Diff for app-page-tur..time.prod.js

Diff too large to display

Diff for app-page.runtime.dev.js

failed to diff

Diff for app-page.runtime.prod.js

Diff too large to display

Diff for pages-turbo...time.prod.js

Diff too large to display

Diff for pages.runtime.dev.js

Diff too large to display

Diff for pages.runtime.prod.js

Diff too large to display

Diff for server.runtime.prod.js

Diff too large to display

Commit: 7edeac4

Ethan-Arrowood requested review from timneutkens, ijjk, shuding, huozhi, feedthejim, ztanner and wyattjoh as code owners April 23, 2024 22:01

ijjk added created-by: Next.js team PRs by the Next.js team type: next labels Apr 23, 2024

Ethan-Arrowood marked this pull request as draft April 24, 2024 16:48

Ethan-Arrowood added 2 commits April 24, 2024 11:06

implement chainStreams

54fee6d

fix implementation, improve types maybe

7edeac4

Ethan-Arrowood force-pushed the experimental-node-streams-support branch from 1d78635 to 4254b93 Compare April 24, 2024 17:06

Ethan-Arrowood mentioned this pull request Apr 24, 2024

Experimental Node.js Streams support #64932

Draft

Ethan-Arrowood force-pushed the 04-23-implement_chainStreams branch from c0db248 to 7edeac4 Compare April 24, 2024 17:07

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Implement Node.js Streams support for `chainStreams` #64948

Implement Node.js Streams support for `chainStreams` #64948

Ethan-Arrowood commented Apr 23, 2024 •

edited by padmaia

Ethan-Arrowood commented Apr 23, 2024

ijjk commented Apr 23, 2024 •

edited

ijjk commented Apr 24, 2024 •

edited

Implement Node.js Streams support for chainStreams #64948

Are you sure you want to change the base?

Implement Node.js Streams support for chainStreams #64948

Conversation

Ethan-Arrowood commented Apr 23, 2024 • edited by padmaia

Ethan-Arrowood commented Apr 23, 2024

ijjk commented Apr 23, 2024 • edited

Failing test suites

ijjk commented Apr 24, 2024 • edited

Stats from current PR

Implement Node.js Streams support for `chainStreams` #64948

Implement Node.js Streams support for `chainStreams` #64948

Ethan-Arrowood commented Apr 23, 2024 •

edited by padmaia

ijjk commented Apr 23, 2024 •

edited

ijjk commented Apr 24, 2024 •

edited