Added EnvHttpProxyAgent to support HTTP_PROXY (#2994)

* feat: added EnvHttpProxyAgent

Closes #1650

* refactor(env-http-proxy-agent): parse NO_PROXY in constructor

* don't use EnvHttpProxyAgent by default

* refactor: use for loop when checking NO_PROXY entries

* feat(env-http-proxy-agent): added httpProxy, httpsProxy & noProxy options

* feat(env-http-proxy-agent): handle changes to NO_PROXY

* docs: added types for EnvHttpProxyAgent

* test: resolve windows issues, mark experimental, update doc

* docs: fix typo

* docs: fetch for EnvHttpProxyAgent

docs/docs/api/EnvHttpProxyAgent.md

@@ -0,0 +1,162 @@
+# Class: EnvHttpProxyAgent
+
+Stability: Experimental.
+
+Extends: `undici.Dispatcher`
+
+EnvHttpProxyAgent automatically reads the proxy configuration from the environment variables `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` and sets up the proxy agents accordingly. When `HTTP_PROXY` and `HTTPS_PROXY` are set, `HTTP_PROXY` is used for HTTP requests and `HTTPS_PROXY` is used for HTTPS requests. If only `HTTP_PROXY` is set, `HTTP_PROXY` is used for both HTTP and HTTPS requests. If only `HTTPS_PROXY` is set, it is only used for HTTPS requests.
+
+`NO_PROXY` is a comma or space-separated list of hostnames that should not be proxied. The list may contain leading wildcard characters (`*`). If `NO_PROXY` is set, the EnvHttpProxyAgent will bypass the proxy for requests to hosts that match the list. If `NO_PROXY` is set to `"*"`, the EnvHttpProxyAgent will bypass the proxy for all requests.
+
+Lower case environment variables are also supported: `http_proxy`, `https_proxy`, and `no_proxy`. However, if both the lower case and upper case environment variables are set, the lower case environment variables will be ignored.
+
+## `new EnvHttpProxyAgent([options])`
+
+Arguments:
+
+* **options** `EnvHttpProxyAgentOptions` (optional) - extends the `Agent` options.
+
+Returns: `EnvHttpProxyAgent`
+
+### Parameter: `EnvHttpProxyAgentOptions`
+
+Extends: [`AgentOptions`](Agent.md#parameter-agentoptions)
+
+* **httpProxy** `string` (optional) - When set, it will override the `HTTP_PROXY` environment variable.
+* **httpsProxy** `string` (optional) - When set, it will override the `HTTPS_PROXY` environment variable.
+* **noProxy** `string` (optional) - When set, it will override the `NO_PROXY` environment variable.
+
+Examples:
+
+```js
+import { EnvHttpProxyAgent } from 'undici'
+
+const envHttpProxyAgent = new EnvHttpProxyAgent()
+// or
+const envHttpProxyAgent = new EnvHttpProxyAgent({ httpProxy: 'my.proxy.server:8080', httpsProxy: 'my.proxy.server:8443', noProxy: 'localhost' })
+```
+
+#### Example - EnvHttpProxyAgent instantiation
+
+This will instantiate the EnvHttpProxyAgent. It will not do anything until registered as the agent to use with requests.
+
+```js
+import { EnvHttpProxyAgent } from 'undici'
+
+const envHttpProxyAgent = new EnvHttpProxyAgent()
+```
+
+#### Example - Basic Proxy Fetch with global agent dispatcher
+
+```js
+import { setGlobalDispatcher, fetch, EnvHttpProxyAgent } from 'undici'
+
+const envHttpProxyAgent = new EnvHttpProxyAgent()
+setGlobalDispatcher(envHttpProxyAgent)
+
+const { status, json } = await fetch('http://localhost:3000/foo')
+
+console.log('response received', status) // response received 200
+
+const data = await json() // data { foo: "bar" }
+```
+
+#### Example - Basic Proxy Request with global agent dispatcher
+
+```js
+import { setGlobalDispatcher, request, EnvHttpProxyAgent } from 'undici'
+
+const envHttpProxyAgent = new EnvHttpProxyAgent()
+setGlobalDispatcher(envHttpProxyAgent)
+
+const { statusCode, body } = await request('http://localhost:3000/foo')
+
+console.log('response received', statusCode) // response received 200
+
+for await (const data of body) {
+  console.log('data', data.toString('utf8')) // data foo
+}
+```
+
+#### Example - Basic Proxy Request with local agent dispatcher
+
+```js
+import { EnvHttpProxyAgent, request } from 'undici'
+
+const envHttpProxyAgent = new EnvHttpProxyAgent()
+
+const {
+  statusCode,
+  body
+} = await request('http://localhost:3000/foo', { dispatcher: envHttpProxyAgent })
+
+console.log('response received', statusCode) // response received 200
+
+for await (const data of body) {
+  console.log('data', data.toString('utf8')) // data foo
+}
+```
+
+#### Example - Basic Proxy Fetch with local agent dispatcher
+
+```js
+import { EnvHttpProxyAgent, fetch } from 'undici'
+
+const envHttpProxyAgent = new EnvHttpProxyAgent()
+
+const {
+  status,
+  json
+} = await fetch('http://localhost:3000/foo', { dispatcher: envHttpProxyAgent })
+
+console.log('response received', status) // response received 200
+
+const data = await json() // data { foo: "bar" }
+```
+
+## Instance Methods
+
+### `EnvHttpProxyAgent.close([callback])`
+
+Implements [`Dispatcher.close([callback])`](Dispatcher.md#dispatcherclosecallback-promise).
+
+### `EnvHttpProxyAgent.destroy([error, callback])`
+
+Implements [`Dispatcher.destroy([error, callback])`](Dispatcher.md#dispatcherdestroyerror-callback-promise).
+
+### `EnvHttpProxyAgent.dispatch(options, handler: AgentDispatchOptions)`
+
+Implements [`Dispatcher.dispatch(options, handler)`](Dispatcher.md#dispatcherdispatchoptions-handler).
+
+#### Parameter: `AgentDispatchOptions`
+
+Extends: [`DispatchOptions`](Dispatcher.md#parameter-dispatchoptions)
+
+* **origin** `string | URL`
+* **maxRedirections** `Integer`.
+
+Implements [`Dispatcher.destroy([error, callback])`](Dispatcher.md#dispatcherdestroyerror-callback-promise).
+
+### `EnvHttpProxyAgent.connect(options[, callback])`
+
+See [`Dispatcher.connect(options[, callback])`](Dispatcher.md#dispatcherconnectoptions-callback).
+
+### `EnvHttpProxyAgent.dispatch(options, handler)`
+
+Implements [`Dispatcher.dispatch(options, handler)`](Dispatcher.md#dispatcherdispatchoptions-handler).
+
+### `EnvHttpProxyAgent.pipeline(options, handler)`
+
+See [`Dispatcher.pipeline(options, handler)`](Dispatcher.md#dispatcherpipelineoptions-handler).
+
+### `EnvHttpProxyAgent.request(options[, callback])`
+
+See [`Dispatcher.request(options [, callback])`](Dispatcher.md#dispatcherrequestoptions-callback).
+
+### `EnvHttpProxyAgent.stream(options, factory[, callback])`
+
+See [`Dispatcher.stream(options, factory[, callback])`](Dispatcher.md#dispatcherstreamoptions-factory-callback).
+
+### `EnvHttpProxyAgent.upgrade(options[, callback])`
+
+See [`Dispatcher.upgrade(options[, callback])`](Dispatcher.md#dispatcherupgradeoptions-callback).

index.js

@@ -6,6 +6,7 @@ const Pool = require('./lib/dispatcher/pool')
 const BalancedPool = require('./lib/dispatcher/balanced-pool')
 const Agent = require('./lib/dispatcher/agent')
 const ProxyAgent = require('./lib/dispatcher/proxy-agent')
+const EnvHttpProxyAgent = require('./lib/dispatcher/env-http-proxy-agent')
 const RetryAgent = require('./lib/dispatcher/retry-agent')
 const errors = require('./lib/core/errors')
 const util = require('./lib/core/util')
@@ -30,6 +31,7 @@ module.exports.Pool = Pool
 module.exports.BalancedPool = BalancedPool
 module.exports.Agent = Agent
 module.exports.ProxyAgent = ProxyAgent
+module.exports.EnvHttpProxyAgent = EnvHttpProxyAgent
 module.exports.RetryAgent = RetryAgent
 module.exports.RetryHandler = RetryHandler
 

lib/core/symbols.js

@@ -60,5 +60,8 @@ module.exports = {
   kConstruct: Symbol('constructable'),
   kListeners: Symbol('listeners'),
   kHTTPContext: Symbol('http context'),
-  kMaxConcurrentStreams: Symbol('max concurrent streams')
+  kMaxConcurrentStreams: Symbol('max concurrent streams'),
+  kNoProxyAgent: Symbol('no proxy agent'),
+  kHttpProxyAgent: Symbol('http proxy agent'),
+  kHttpsProxyAgent: Symbol('https proxy agent')
 }

lib/dispatcher/dispatcher-base.js

@@ -6,10 +6,8 @@ const {
   ClientClosedError,
   InvalidArgumentError
 } = require('../core/errors')
-const { kDestroy, kClose, kDispatch, kInterceptors } = require('../core/symbols')
+const { kDestroy, kClose, kClosed, kDestroyed, kDispatch, kInterceptors } = require('../core/symbols')
 
-const kDestroyed = Symbol('destroyed')
-const kClosed = Symbol('closed')
 const kOnDestroyed = Symbol('onDestroyed')
 const kOnClosed = Symbol('onClosed')
 const kInterceptedDispatch = Symbol('Intercepted Dispatch')

lib/dispatcher/env-http-proxy-agent.js

@@ -0,0 +1,160 @@
+'use strict'
+
+const DispatcherBase = require('./dispatcher-base')
+const { kClose, kDestroy, kClosed, kDestroyed, kDispatch, kNoProxyAgent, kHttpProxyAgent, kHttpsProxyAgent } = require('../core/symbols')
+const ProxyAgent = require('./proxy-agent')
+const Agent = require('./agent')
+
+const DEFAULT_PORTS = {
+  'http:': 80,
+  'https:': 443
+}
+
+let experimentalWarned = false
+
+class EnvHttpProxyAgent extends DispatcherBase {
+  #noProxyValue = null
+  #noProxyEntries = null
+  #opts = null
+
+  constructor (opts = {}) {
+    super()
+    this.#opts = opts
+
+    if (!experimentalWarned) {
+      experimentalWarned = true
+      process.emitWarning('EnvHttpProxyAgent is experimental, expect them to change at any time.', {
+        code: 'UNDICI-EHPA'
+      })
+    }
+
+    const { httpProxy, httpsProxy, noProxy, ...agentOpts } = opts
+
+    this[kNoProxyAgent] = new Agent(agentOpts)
+
+    const HTTP_PROXY = httpProxy ?? process.env.HTTP_PROXY ?? process.env.http_proxy
+    if (HTTP_PROXY) {
+      this[kHttpProxyAgent] = new ProxyAgent({ ...agentOpts, uri: HTTP_PROXY })
+    } else {
+      this[kHttpProxyAgent] = this[kNoProxyAgent]
+    }
+
+    const HTTPS_PROXY = httpsProxy ?? process.env.HTTPS_PROXY ?? process.env.https_proxy
+    if (HTTPS_PROXY) {
+      this[kHttpsProxyAgent] = new ProxyAgent({ ...agentOpts, uri: HTTPS_PROXY })
+    } else {
+      this[kHttpsProxyAgent] = this[kHttpProxyAgent]
+    }
+
+    this.#parseNoProxy()
+  }
+
+  [kDispatch] (opts, handler) {
+    const url = new URL(opts.origin)
+    const agent = this.#getProxyAgentForUrl(url)
+    return agent.dispatch(opts, handler)
+  }
+
+  async [kClose] () {
+    await this[kNoProxyAgent].close()
+    if (!this[kHttpProxyAgent][kClosed]) {
+      await this[kHttpProxyAgent].close()
+    }
+    if (!this[kHttpsProxyAgent][kClosed]) {
+      await this[kHttpsProxyAgent].close()
+    }
+  }
+
+  async [kDestroy] (err) {
+    await this[kNoProxyAgent].destroy(err)
+    if (!this[kHttpProxyAgent][kDestroyed]) {
+      await this[kHttpProxyAgent].destroy(err)
+    }
+    if (!this[kHttpsProxyAgent][kDestroyed]) {
+      await this[kHttpsProxyAgent].destroy(err)
+    }
+  }
+
+  #getProxyAgentForUrl (url) {
+    let { protocol, host: hostname, port } = url
+
+    // Stripping ports in this way instead of using parsedUrl.hostname to make
+    // sure that the brackets around IPv6 addresses are kept.
+    hostname = hostname.replace(/:\d*$/, '').toLowerCase()
+    port = Number.parseInt(port, 10) || DEFAULT_PORTS[protocol] || 0
+    if (!this.#shouldProxy(hostname, port)) {
+      return this[kNoProxyAgent]
+    }
+    if (protocol === 'https:') {
+      return this[kHttpsProxyAgent]
+    }
+    return this[kHttpProxyAgent]
+  }
+
+  #shouldProxy (hostname, port) {
+    if (this.#noProxyChanged) {
+      this.#parseNoProxy()
+    }
+
+    if (this.#noProxyEntries.length === 0) {
+      return true // Always proxy if NO_PROXY is not set or empty.
+    }
+    if (this.#noProxyValue === '*') {
+      return false // Never proxy if wildcard is set.
+    }
+
+    for (let i = 0; i < this.#noProxyEntries.length; i++) {
+      const entry = this.#noProxyEntries[i]
+      if (entry.port && entry.port !== port) {
+        continue // Skip if ports don't match.
+      }
+      if (!/^[.*]/.test(entry.hostname)) {
+        // No wildcards, so don't proxy only if there is not an exact match.
+        if (hostname === entry.hostname) {
+          return false
+        }
+      } else {
+        // Don't proxy if the hostname ends with the no_proxy host.
+        if (hostname.endsWith(entry.hostname.replace(/^\*/, ''))) {
+          return false
+        }
+      }
+    }
+
+    return true
+  }
+
+  #parseNoProxy () {
+    const noProxyValue = this.#opts.noProxy ?? this.#noProxyEnv
+    const noProxySplit = noProxyValue.split(/[,\s]/)
+    const noProxyEntries = []
+
+    for (let i = 0; i < noProxySplit.length; i++) {
+      const entry = noProxySplit[i]
+      if (!entry) {
+        continue
+      }
+      const parsed = entry.match(/^(.+):(\d+)$/)
+      noProxyEntries.push({
+        hostname: (parsed ? parsed[1] : entry).toLowerCase(),
+        port: parsed ? Number.parseInt(parsed[2], 10) : 0
+      })
+    }
+
+    this.#noProxyValue = noProxyValue
+    this.#noProxyEntries = noProxyEntries
+  }
+
+  get #noProxyChanged () {
+    if (this.#opts.noProxy !== undefined) {
+      return false
+    }
+    return this.#noProxyValue !== this.#noProxyEnv
+  }
+
+  get #noProxyEnv () {
+    return process.env.NO_PROXY ?? process.env.no_proxy ?? ''
+  }
+}
+
+module.exports = EnvHttpProxyAgent

package.json

@@ -118,6 +118,7 @@
     "node-forge": "^1.3.1",
     "pre-commit": "^1.2.2",
     "proxy": "^2.1.1",
+    "sinon": "^17.0.1",
     "snazzy": "^9.0.0",
     "standard": "^17.0.0",
     "tsd": "^0.31.0",