add experimental.urlImports option (#30079)

Co-authored-by: Lee Robinson <me@leerob.io>
Co-authored-by: Rich Haines <hello@richardhaines.dev>

docs/api-reference/next.config.js/url-imports.md

@@ -0,0 +1,37 @@
+---
+description: Configure Next.js to allow importing modules from external URLs.
+---
+
+# URL imports
+
+URL Imports are an experimental feature that allows you to import modules directly from external servers (instead of from the local disk).
+
+To opt-in, add the allowed URL prefixes inside `next.config.js`:
+
+```js
+module.exports = {
+  experimental: {
+    urlImports: ['https://example.com/modules/'],
+  },
+}
+```
+
+Then, you can import modules directly from URLs:
+
+```js
+import { a, b, c } from 'https://example.com/modules/'
+```
+
+URL Imports can be used everywhere normal package imports can be used.
+
+## Lockfile
+
+When using URL imports, Next.js will create a lockfile in the `next.lock` directory.
+This directory is intended to be committed to Git and should **not be included** in your `.gitignore` file.
+
+- When running `next dev`, Next.js will download and add all newly discovered URL Imports to your lockfile
+- When running `next build`, Next.js will use only the lockfile to build the application for production
+
+Typically, no network requests are needed and any outdated lockfile will cause the build to fail.
+One exception is resources that respond with `Cache-Control: no-cache`.
+These resources will have a `no-cache` entry in the lockfile and will always be fetched from the network on each build.

docs/manifest.json

@@ -387,6 +387,10 @@
             {
               "title": "React Strict Mode",
               "path": "/docs/api-reference/next.config.js/react-strict-mode.md"
+            },
+            {
+              "title": "URL Imports",
+              "path": "/docs/api-reference/next.config.js/url-imports.md"
             }
           ]
         }

packages/next/build/webpack-config.ts

@@ -1326,6 +1326,19 @@ export default async function getBaseWebpackConfig(
   webpack5Config.experiments = {
     layers: true,
     cacheUnaffected: true,
+    buildHttp: Array.isArray(config.experimental.urlImports)
+      ? {
+          allowedUris: config.experimental.urlImports,
+          cacheLocation: path.join(dir, 'next.lock/data'),
+          lockfileLocation: path.join(dir, 'next.lock/lock.json'),
+        }
+      : config.experimental.urlImports
+      ? {
+          cacheLocation: path.join(dir, 'next.lock/data'),
+          lockfileLocation: path.join(dir, 'next.lock/lock.json'),
+          ...config.experimental.urlImports,
+        }
+      : undefined,
   }
 
   webpack5Config.module!.parser = {

packages/next/server/config-shared.ts

@@ -1,4 +1,5 @@
 import os from 'os'
+import type webpack5 from 'webpack5'
 import { Header, Redirect, Rewrite } from '../lib/load-custom-routes'
 import {
   ImageConfig,
@@ -147,6 +148,7 @@ export type NextConfig = { [key: string]: any } & {
     concurrentFeatures?: boolean
     serverComponents?: boolean
     fullySpecified?: boolean
+    urlImports?: NonNullable<webpack5.Configuration['experiments']>['buildHttp']
   }
 }
 

test/integration/url-imports/.gitignore

@@ -0,0 +1,3 @@
+# this is only needed for the test case
+# Do not ignore that in real apps
+next.lock

test/integration/url-imports/next.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  experimental: {
+    urlImports: ['http://localhost:12345/'],
+  },
+}

test/integration/url-imports/pages/api/value.js

@@ -0,0 +1,5 @@
+import value from 'http://localhost:12345/value4.js'
+
+export default (req, res) => {
+  res.json({ value: value })
+}

test/integration/url-imports/pages/ssg.js

@@ -0,0 +1,17 @@
+import value from 'http://localhost:12345/value1.js'
+
+export async function getStaticProps() {
+  return {
+    props: {
+      value,
+    },
+  }
+}
+
+export default function Index({ value: staticValue }) {
+  return (
+    <div>
+      Hello {staticValue}+{value}
+    </div>
+  )
+}

test/integration/url-imports/pages/ssr.js

@@ -0,0 +1,17 @@
+import value from 'http://localhost:12345/value2.js'
+
+export function getServerSideProps() {
+  return {
+    props: {
+      value,
+    },
+  }
+}
+
+export default function Index({ value: serverValue }) {
+  return (
+    <div>
+      Hello {serverValue}+{value}
+    </div>
+  )
+}

test/integration/url-imports/pages/static.js

@@ -0,0 +1,9 @@
+import value from 'http://localhost:12345/value3.js'
+
+export default function Index(props) {
+  return (
+    <div>
+      Hello {value}+{value}
+    </div>
+  )
+}

test/integration/url-imports/source/value1.js

@@ -0,0 +1 @@
+export default 42 // 1

test/integration/url-imports/source/value2.js

@@ -0,0 +1 @@
+export default 42 // 2

test/integration/url-imports/source/value3.js

@@ -0,0 +1 @@
+export default 42 // 3

test/integration/url-imports/source/value4.js

@@ -0,0 +1 @@
+export default 42 // 4

test/integration/url-imports/test/index.test.js

@@ -0,0 +1,89 @@
+/* eslint-disable no-loop-func */
+/* eslint-env jest */
+
+import fs from 'fs-extra'
+import { join } from 'path'
+import {
+  nextBuild,
+  findPort,
+  nextStart,
+  killApp,
+  renderViaHTTP,
+  fetchViaHTTP,
+  launchApp,
+  getBrowserBodyText,
+  check,
+  startStaticServer,
+  stopApp,
+} from 'next-test-utils'
+import webdriver from 'next-webdriver'
+
+jest.setTimeout(1000 * 60 * 2)
+const appDir = join(__dirname, '../')
+
+describe(`Handle url imports`, () => {
+  let staticServer
+  let staticServerPort
+  beforeAll(async () => {
+    await fs.remove(join(appDir, 'next.lock'))
+    staticServerPort = 12345
+    staticServer = await startStaticServer(
+      join(appDir, 'source'),
+      undefined,
+      staticServerPort
+    )
+  })
+  afterAll(async () => {
+    await stopApp(staticServer)
+  })
+
+  for (const dev of [true, false]) {
+    describe(dev ? 'with next dev' : 'with next build', () => {
+      let appPort
+      let app
+      beforeAll(async () => {
+        await fs.remove(join(appDir, '.next'))
+        if (dev) {
+          appPort = await findPort()
+          app = await launchApp(appDir, appPort)
+        } else {
+          await nextBuild(appDir)
+          appPort = await findPort()
+          app = await nextStart(appDir, appPort)
+        }
+      })
+      afterAll(async () => {
+        await killApp(app)
+      })
+      const expectedServer = /Hello <!-- -->42<!-- -->\+<!-- -->42/
+      const expectedClient = new RegExp(
+        expectedServer.source.replace(/<!-- -->/g, '')
+      )
+
+      for (const page of ['/static', '/ssr', '/ssg']) {
+        it(`should render the ${page} page`, async () => {
+          const html = await renderViaHTTP(appPort, page)
+          expect(html).toMatch(expectedServer)
+        })
+
+        it(`should client-render the ${page} page`, async () => {
+          let browser
+          try {
+            browser = await webdriver(appPort, page)
+            await check(() => getBrowserBodyText(browser), expectedClient)
+          } finally {
+            await browser.close()
+          }
+        })
+      }
+
+      it('should respond on value api', async () => {
+        const data = await fetchViaHTTP(appPort, '/api/value').then(
+          (res) => res.ok && res.json()
+        )
+
+        expect(data).toEqual({ value: 42 })
+      })
+    })
+  }
+})

test/lib/next-test-utils.js

@@ -386,7 +386,7 @@ export function waitFor(millis) {
   return new Promise((resolve) => setTimeout(resolve, millis))
 }
 
-export async function startStaticServer(dir, notFoundFile) {
+export async function startStaticServer(dir, notFoundFile, fixedPort) {
   const app = express()
   const server = http.createServer(app)
   app.use(express.static(dir))
@@ -397,7 +397,7 @@ export async function startStaticServer(dir, notFoundFile) {
     })
   }
 
-  await promiseCall(server, 'listen')
+  await promiseCall(server, 'listen', fixedPort)
   return server
 }