feat: support custom highWaterMark

closes #435

README.md

@@ -75,6 +75,7 @@ console.log('status: %s, body size: %d, headers: %j', res.status, data.length, r
   - ***compressed*** Boolean - Accept `gzip, br` response content and auto decode it, default is `false`.
   - ***timing*** Boolean - Enable timing or not, default is `false`.
   - ***socketPath*** String | null - request a unix socket service, default is `null`.
+  - ***highWaterMark*** Number - default is `67108864`, 64 KiB.
 
 #### Options: `options.data`
 

package.json

@@ -73,7 +73,7 @@
     "@types/busboy": "^1.5.0",
     "@types/default-user-agent": "^1.0.0",
     "@types/mime-types": "^2.1.1",
-    "@types/node": "^18.11.18",
+    "@types/node": "^20.2.1",
     "@types/pump": "^1.1.1",
     "@types/selfsigned": "^2.0.1",
     "@types/tar-stream": "^2.2.2",

src/HttpClient.ts

@@ -316,10 +316,12 @@ export class HttpClient extends EventEmitter {
         opaque: internalOpaque,
         dispatcher: args.dispatcher ?? this.#dispatcher,
       };
+      if (typeof args.highWaterMark === 'number') {
+        requestOptions.highWaterMark = args.highWaterMark;
+      }
       if (typeof args.reset === 'boolean') {
         requestOptions.reset = args.reset;
       }
-
       if (args.followRedirect === false) {
         requestOptions.maxRedirections = 0;
       }

src/Request.ts

@@ -125,11 +125,12 @@ export type RequestOptions = {
    * request dispatcher, default is getGlobalDispatcher()
    */
   dispatcher?: Dispatcher;
-
   /**
    * unix domain socket file path
    */
   socketPath?: string | null;
   /** Whether the request should stablish a keep-alive or not. Default `undefined` */
   reset?: boolean;
+  /** Default: `64 KiB` */
+  highWaterMark?: number;
 };

test/options.streaming.test.ts

@@ -140,4 +140,32 @@ describe('options.streaming.test.ts', () => {
     const bytes = await readableToBytes(response.res);
     assert.equal(bytes.length, 1024102400);
   });
+
+  it('should save big streaming response with highWaterMark', async () => {
+    let start = Date.now();
+    const size = 1024 * 1024 * 100;
+    const response1 = await urllib.request(`${_url}mock-bytes?size=${size}`, {
+      streaming: true,
+      // highWaterMark: 64 * 1024,
+    });
+    assert.equal(response1.status, 200);
+    assert(isReadable(response1.res as any));
+    const bytes1 = await readableToBytes(response1.res);
+    assert.equal(bytes1.length, size);
+    const use1 = Date.now() - start;
+    console.log('highWaterMark 64KB use %dms', use1);
+
+    start = Date.now();
+    const response2 = await urllib.request(`${_url}mock-bytes?size=${size}`, {
+      streaming: true,
+      highWaterMark: 128 * 1024,
+    });
+    assert.equal(response2.status, 200);
+    assert(isReadable(response2.res as any));
+    const bytes2 = await readableToBytes(response2.res);
+    assert.equal(bytes2.length, size);
+    const use2 = Date.now() - start;
+    console.log('highWaterMark 128KB use %dms', use2);
+    assert(use2 < use1);
+  });
 });