Add initial standalone build handling (#31003)

* Add initial standalone handling

* apply suggestions

* Apply suggestions from code review

Co-authored-by: Steven <steven@ceriously.com>

packages/next/build/index.ts

@@ -91,6 +91,7 @@ import {
   printTreeView,
   getCssFilePaths,
   getUnresolvedModuleFromError,
+  copyTracedFiles,
   isReservedPage,
   isCustomErrorPage,
 } from './utils'
@@ -103,6 +104,7 @@ import isError, { NextError } from '../lib/is-error'
 import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin'
 import { MiddlewareManifest } from './webpack/plugins/middleware-plugin'
 import type { webpack5 as webpack } from 'next/dist/compiled/webpack/webpack'
+import { recursiveCopy } from '../lib/recursive-copy'
 
 export type SsgRoute = {
   initialRevalidateSeconds: number | false
@@ -552,6 +554,7 @@ export default async function build(
           path.relative(distDir, manifestPath),
           BUILD_MANIFEST,
           PRERENDER_MANIFEST,
+          path.join(SERVER_DIRECTORY, MIDDLEWARE_MANIFEST),
           hasServerComponents
             ? path.join(SERVER_DIRECTORY, MIDDLEWARE_FLIGHT_MANIFEST + '.js')
             : null,
@@ -1362,6 +1365,23 @@ export default async function build(
       'utf8'
     )
 
+    const outputFileTracingRoot =
+      config.experimental.outputFileTracingRoot || dir
+
+    if (config.experimental.outputStandalone) {
+      await nextBuildSpan
+        .traceChild('copy-traced-files')
+        .traceAsyncFn(async () => {
+          await copyTracedFiles(
+            dir,
+            distDir,
+            pageKeys,
+            outputFileTracingRoot,
+            requiredServerFiles.config
+          )
+        })
+    }
+
     const finalPrerenderRoutes: { [route: string]: SsgRoute } = {}
     const tbdPrerenderRoutes: string[] = []
     let ssgNotFoundPaths: string[] = []
@@ -1957,6 +1977,33 @@ export default async function build(
       return Promise.reject(err)
     })
 
+    if (config.experimental.outputStandalone) {
+      for (const file of [
+        ...requiredServerFiles.files,
+        path.join(config.distDir, SERVER_FILES_MANIFEST),
+      ]) {
+        const filePath = path.join(dir, file)
+        await promises.copyFile(
+          filePath,
+          path.join(
+            distDir,
+            'standalone',
+            path.relative(outputFileTracingRoot, filePath)
+          )
+        )
+      }
+      await recursiveCopy(
+        path.join(distDir, SERVER_DIRECTORY, 'pages'),
+        path.join(
+          distDir,
+          'standalone',
+          path.relative(outputFileTracingRoot, distDir),
+          SERVER_DIRECTORY,
+          'pages'
+        )
+      )
+    }
+
     staticPages.forEach((pg) => allStaticPages.add(pg))
     pageInfos.forEach((info: PageInfo, key: string) => {
       allPageInfos.set(key, info)

packages/next/build/utils.ts

@@ -24,7 +24,10 @@ import { isDynamicRoute } from '../shared/lib/router/utils/is-dynamic'
 import escapePathDelimiters from '../shared/lib/router/utils/escape-path-delimiters'
 import { findPageFile } from '../server/lib/find-page-file'
 import { GetStaticPaths, PageConfig } from 'next/types'
-import { denormalizePagePath } from '../server/normalize-page-path'
+import {
+  denormalizePagePath,
+  normalizePagePath,
+} from '../server/normalize-page-path'
 import { BuildManifest } from '../server/get-page-files'
 import { removePathTrailingSlash } from '../client/normalize-trailing-slash'
 import { UnwrapPromise } from '../lib/coalesced-function'
@@ -35,6 +38,8 @@ import { trace } from '../trace'
 import { setHttpAgentOptions } from '../server/config'
 import { NextConfigComplete } from '../server/config-shared'
 import isError from '../lib/is-error'
+import { recursiveDelete } from '../lib/recursive-delete'
+import { Sema } from 'next/dist/compiled/async-sema'
 
 const { builtinModules } = require('module')
 const RESERVED_PAGE = /^\/(_app|_error|_document|api(\/|$))/
@@ -1146,6 +1151,112 @@ export function getUnresolvedModuleFromError(
   return builtinModules.find((item: string) => item === moduleName)
 }
 
+export async function copyTracedFiles(
+  dir: string,
+  distDir: string,
+  pageKeys: string[],
+  tracingRoot: string,
+  serverConfig: { [key: string]: any }
+) {
+  const outputPath = path.join(distDir, 'standalone')
+  const copiedFiles = new Set()
+  await recursiveDelete(outputPath)
+
+  async function handleTraceFiles(traceFilePath: string) {
+    const traceData = JSON.parse(await fs.readFile(traceFilePath, 'utf8')) as {
+      files: string[]
+    }
+    const copySema = new Sema(10, { capacity: traceData.files.length })
+    const traceFileDir = path.dirname(traceFilePath)
+
+    await Promise.all(
+      traceData.files.map(async (relativeFile) => {
+        await copySema.acquire()
+
+        const tracedFilePath = path.join(traceFileDir, relativeFile)
+        const fileOutputPath = path.join(
+          outputPath,
+          path.relative(tracingRoot, tracedFilePath)
+        )
+
+        if (!copiedFiles.has(fileOutputPath)) {
+          copiedFiles.add(fileOutputPath)
+
+          await fs.mkdir(path.dirname(fileOutputPath), { recursive: true })
+          const symlink = await fs.readlink(tracedFilePath).catch(() => null)
+
+          if (symlink) {
+            console.log('symlink', path.relative(tracingRoot, symlink))
+            await fs.symlink(
+              path.relative(tracingRoot, symlink),
+              fileOutputPath
+            )
+          } else {
+            await fs.copyFile(tracedFilePath, fileOutputPath)
+          }
+        }
+
+        await copySema.release()
+      })
+    )
+  }
+
+  for (const page of pageKeys) {
+    const pageFile = path.join(
+      distDir,
+      'server',
+      'pages',
+      `${normalizePagePath(page)}.js`
+    )
+    const pageTraceFile = `${pageFile}.nft.json`
+    await handleTraceFiles(pageTraceFile)
+  }
+  await handleTraceFiles(path.join(distDir, 'next-server.js.nft.json'))
+  const serverOutputPath = path.join(
+    outputPath,
+    path.relative(tracingRoot, dir),
+    'server.js'
+  )
+  await fs.writeFile(
+    serverOutputPath,
+    `
+process.env.NODE_ENV = 'production'
+process.chdir(__dirname)
+const NextServer = require('next/dist/server/next-server').default
+const http = require('http')
+const path = require('path')
+
+const nextServer = new NextServer({
+  dir: path.join(__dirname),
+  dev: false,
+  conf: ${JSON.stringify({
+    ...serverConfig,
+    distDir: `./${path.relative(dir, distDir)}`,
+  })},
+})
+
+const handler = nextServer.getRequestHandler()
+
+const server = http.createServer(async (req, res) => {
+  try {
+    await handler(req, res)
+  } catch (err) {
+    console.error(err);
+    res.statusCode = 500
+    res.end('internal server error')
+  }
+})
+const currentPort = process.env.PORT || 3000
+server.listen(currentPort, (err) => {
+  if (err) {
+    console.error("Failed to start server", err)
+    process.exit(1)
+  }
+  console.log("Listening on port", currentPort)
+})
+    `
+  )
+}
 export function isReservedPage(page: string) {
   return RESERVED_PAGE.test(page)
 }

packages/next/lib/recursive-copy.ts

@@ -47,7 +47,7 @@ export async function recursiveCopy(
 
     if (isDirectory) {
       try {
-        await promises.mkdir(target)
+        await promises.mkdir(target, { recursive: true })
       } catch (err) {
         // do not throw `folder already exists` errors
         if (isError(err) && err.code !== 'EEXIST') {

packages/next/lib/recursive-delete.ts

@@ -1,13 +1,17 @@
 import { Dirent, promises } from 'fs'
-import { join } from 'path'
+import { join, isAbsolute, dirname } from 'path'
 import { promisify } from 'util'
 import isError from './is-error'
 
 const sleep = promisify(setTimeout)
 
-const unlinkFile = async (p: string, t = 1): Promise<void> => {
+const unlinkPath = async (p: string, isDir = false, t = 1): Promise<void> => {
   try {
-    await promises.unlink(p)
+    if (isDir) {
+      await promises.rmdir(p)
+    } else {
+      await promises.unlink(p)
+    }
   } catch (e) {
     const code = isError(e) && e.code
     if (
@@ -18,7 +22,7 @@ const unlinkFile = async (p: string, t = 1): Promise<void> => {
       t < 3
     ) {
       await sleep(t * 100)
-      return unlinkFile(p, t++)
+      return unlinkPath(p, isDir, t++)
     }
 
     if (code === 'ENOENT') {
@@ -58,19 +62,29 @@ export async function recursiveDelete(
       // readdir does not follow symbolic links
       // if part is a symbolic link, follow it using stat
       let isDirectory = part.isDirectory()
-      if (part.isSymbolicLink()) {
-        const stats = await promises.stat(absolutePath)
-        isDirectory = stats.isDirectory()
+      const isSymlink = part.isSymbolicLink()
+
+      if (isSymlink) {
+        const linkPath = await promises.readlink(absolutePath)
+
+        try {
+          const stats = await promises.stat(
+            isAbsolute(linkPath)
+              ? linkPath
+              : join(dirname(absolutePath), linkPath)
+          )
+          isDirectory = stats.isDirectory()
+        } catch (_) {}
       }
 
       const pp = join(previousPath, part.name)
-      if (isDirectory && (!exclude || !exclude.test(pp))) {
-        await recursiveDelete(absolutePath, exclude, pp)
-        return promises.rmdir(absolutePath)
-      }
+      const isNotExcluded = !exclude || !exclude.test(pp)
 
-      if (!exclude || !exclude.test(pp)) {
-        return unlinkFile(absolutePath)
+      if (isNotExcluded) {
+        if (isDirectory) {
+          await recursiveDelete(absolutePath, exclude, pp)
+        }
+        return unlinkPath(absolutePath, !isSymlink && isDirectory)
       }
     })
   )

packages/next/server/config-shared.ts

@@ -157,6 +157,7 @@ export type NextConfig = { [key: string]: any } & {
     fullySpecified?: boolean
     urlImports?: NonNullable<webpack5.Configuration['experiments']>['buildHttp']
     outputFileTracingRoot?: string
+    outputStandalone?: boolean
   }
 }
 
@@ -239,6 +240,7 @@ export const defaultConfig: NextConfig = {
     serverComponents: false,
     fullySpecified: false,
     outputFileTracingRoot: process.env.NEXT_PRIVATE_OUTPUT_TRACE_ROOT || '',
+    outputStandalone: !!process.env.NEXT_PRIVATE_STANDALONE,
   },
   future: {
     strictPostcssConfiguration: false,

packages/next/server/next-server.ts

@@ -2551,11 +2551,13 @@ export default class Server {
     }
 
     let nextFilesStatic: string[] = []
-    nextFilesStatic = !this.minimalMode
-      ? recursiveReadDirSync(join(this.distDir, 'static')).map((f) =>
-          join('.', relative(this.dir, this.distDir), 'static', f)
-        )
-      : []
+
+    nextFilesStatic =
+      !this.minimalMode && fs.existsSync(join(this.distDir, 'static'))
+        ? recursiveReadDirSync(join(this.distDir, 'static')).map((f) =>
+            join('.', relative(this.dir, this.distDir), 'static', f)
+          )
+        : []
 
     return (this._validFilesystemPathSet = new Set<string>([
       ...nextFilesStatic,