diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts index aaff62ec7fca273..d826fbd436cf2da 100644 --- a/packages/next/build/index.ts +++ b/packages/next/build/index.ts @@ -91,6 +91,7 @@ import { printTreeView, getCssFilePaths, getUnresolvedModuleFromError, + copyTracedFiles, isReservedPage, isCustomErrorPage, } from './utils' @@ -103,6 +104,7 @@ import isError, { NextError } from '../lib/is-error' import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin' import { MiddlewareManifest } from './webpack/plugins/middleware-plugin' import type { webpack5 as webpack } from 'next/dist/compiled/webpack/webpack' +import { recursiveCopy } from '../lib/recursive-copy' export type SsgRoute = { initialRevalidateSeconds: number | false @@ -552,6 +554,7 @@ export default async function build( path.relative(distDir, manifestPath), BUILD_MANIFEST, PRERENDER_MANIFEST, + path.join(SERVER_DIRECTORY, MIDDLEWARE_MANIFEST), hasServerComponents ? path.join(SERVER_DIRECTORY, MIDDLEWARE_FLIGHT_MANIFEST + '.js') : null, @@ -1362,6 +1365,23 @@ export default async function build( 'utf8' ) + const outputFileTracingRoot = + config.experimental.outputFileTracingRoot || dir + + if (config.experimental.outputStandalone) { + await nextBuildSpan + .traceChild('copy-traced-files') + .traceAsyncFn(async () => { + await copyTracedFiles( + dir, + distDir, + pageKeys, + outputFileTracingRoot, + requiredServerFiles.config + ) + }) + } + const finalPrerenderRoutes: { [route: string]: SsgRoute } = {} const tbdPrerenderRoutes: string[] = [] let ssgNotFoundPaths: string[] = [] @@ -1957,6 +1977,33 @@ export default async function build( return Promise.reject(err) }) + if (config.experimental.outputStandalone) { + for (const file of [ + ...requiredServerFiles.files, + path.join(config.distDir, SERVER_FILES_MANIFEST), + ]) { + const filePath = path.join(dir, file) + await promises.copyFile( + filePath, + path.join( + distDir, + 'standalone', + path.relative(outputFileTracingRoot, filePath) + ) + ) + } + await recursiveCopy( + path.join(distDir, SERVER_DIRECTORY, 'pages'), + path.join( + distDir, + 'standalone', + path.relative(outputFileTracingRoot, distDir), + SERVER_DIRECTORY, + 'pages' + ) + ) + } + staticPages.forEach((pg) => allStaticPages.add(pg)) pageInfos.forEach((info: PageInfo, key: string) => { allPageInfos.set(key, info) diff --git a/packages/next/build/utils.ts b/packages/next/build/utils.ts index 54a48944011227d..4eaef3e0cf7a906 100644 --- a/packages/next/build/utils.ts +++ b/packages/next/build/utils.ts @@ -24,7 +24,10 @@ import { isDynamicRoute } from '../shared/lib/router/utils/is-dynamic' import escapePathDelimiters from '../shared/lib/router/utils/escape-path-delimiters' import { findPageFile } from '../server/lib/find-page-file' import { GetStaticPaths, PageConfig } from 'next/types' -import { denormalizePagePath } from '../server/normalize-page-path' +import { + denormalizePagePath, + normalizePagePath, +} from '../server/normalize-page-path' import { BuildManifest } from '../server/get-page-files' import { removePathTrailingSlash } from '../client/normalize-trailing-slash' import { UnwrapPromise } from '../lib/coalesced-function' @@ -35,6 +38,8 @@ import { trace } from '../trace' import { setHttpAgentOptions } from '../server/config' import { NextConfigComplete } from '../server/config-shared' import isError from '../lib/is-error' +import { recursiveDelete } from '../lib/recursive-delete' +import { Sema } from 'next/dist/compiled/async-sema' const { builtinModules } = require('module') const RESERVED_PAGE = /^\/(_app|_error|_document|api(\/|$))/ @@ -1146,6 +1151,112 @@ export function getUnresolvedModuleFromError( return builtinModules.find((item: string) => item === moduleName) } +export async function copyTracedFiles( + dir: string, + distDir: string, + pageKeys: string[], + tracingRoot: string, + serverConfig: { [key: string]: any } +) { + const outputPath = path.join(distDir, 'standalone') + const copiedFiles = new Set() + await recursiveDelete(outputPath) + + async function handleTraceFiles(traceFilePath: string) { + const traceData = JSON.parse(await fs.readFile(traceFilePath, 'utf8')) as { + files: string[] + } + const copySema = new Sema(10, { capacity: traceData.files.length }) + const traceFileDir = path.dirname(traceFilePath) + + await Promise.all( + traceData.files.map(async (relativeFile) => { + await copySema.acquire() + + const tracedFilePath = path.join(traceFileDir, relativeFile) + const fileOutputPath = path.join( + outputPath, + path.relative(tracingRoot, tracedFilePath) + ) + + if (!copiedFiles.has(fileOutputPath)) { + copiedFiles.add(fileOutputPath) + + await fs.mkdir(path.dirname(fileOutputPath), { recursive: true }) + const symlink = await fs.readlink(tracedFilePath).catch(() => null) + + if (symlink) { + console.log('symlink', path.relative(tracingRoot, symlink)) + await fs.symlink( + path.relative(tracingRoot, symlink), + fileOutputPath + ) + } else { + await fs.copyFile(tracedFilePath, fileOutputPath) + } + } + + await copySema.release() + }) + ) + } + + for (const page of pageKeys) { + const pageFile = path.join( + distDir, + 'server', + 'pages', + `${normalizePagePath(page)}.js` + ) + const pageTraceFile = `${pageFile}.nft.json` + await handleTraceFiles(pageTraceFile) + } + await handleTraceFiles(path.join(distDir, 'next-server.js.nft.json')) + const serverOutputPath = path.join( + outputPath, + path.relative(tracingRoot, dir), + 'server.js' + ) + await fs.writeFile( + serverOutputPath, + ` +process.env.NODE_ENV = 'production' +process.chdir(__dirname) +const NextServer = require('next/dist/server/next-server').default +const http = require('http') +const path = require('path') + +const nextServer = new NextServer({ + dir: path.join(__dirname), + dev: false, + conf: ${JSON.stringify({ + ...serverConfig, + distDir: `./${path.relative(dir, distDir)}`, + })}, +}) + +const handler = nextServer.getRequestHandler() + +const server = http.createServer(async (req, res) => { + try { + await handler(req, res) + } catch (err) { + console.error(err); + res.statusCode = 500 + res.end('internal server error') + } +}) +const currentPort = process.env.PORT || 3000 +server.listen(currentPort, (err) => { + if (err) { + console.error("Failed to start server", err) + process.exit(1) + } + console.log("Listening on port", currentPort) +}) + ` + ) +} export function isReservedPage(page: string) { return RESERVED_PAGE.test(page) } diff --git a/packages/next/lib/recursive-copy.ts b/packages/next/lib/recursive-copy.ts index 81302ddb75fa443..df9d163de5d110b 100644 --- a/packages/next/lib/recursive-copy.ts +++ b/packages/next/lib/recursive-copy.ts @@ -47,7 +47,7 @@ export async function recursiveCopy( if (isDirectory) { try { - await promises.mkdir(target) + await promises.mkdir(target, { recursive: true }) } catch (err) { // do not throw `folder already exists` errors if (isError(err) && err.code !== 'EEXIST') { diff --git a/packages/next/lib/recursive-delete.ts b/packages/next/lib/recursive-delete.ts index c23dbce4de6487b..76ed97712fb345d 100644 --- a/packages/next/lib/recursive-delete.ts +++ b/packages/next/lib/recursive-delete.ts @@ -1,13 +1,17 @@ import { Dirent, promises } from 'fs' -import { join } from 'path' +import { join, isAbsolute, dirname } from 'path' import { promisify } from 'util' import isError from './is-error' const sleep = promisify(setTimeout) -const unlinkFile = async (p: string, t = 1): Promise => { +const unlinkPath = async (p: string, isDir = false, t = 1): Promise => { try { - await promises.unlink(p) + if (isDir) { + await promises.rmdir(p) + } else { + await promises.unlink(p) + } } catch (e) { const code = isError(e) && e.code if ( @@ -18,7 +22,7 @@ const unlinkFile = async (p: string, t = 1): Promise => { t < 3 ) { await sleep(t * 100) - return unlinkFile(p, t++) + return unlinkPath(p, isDir, t++) } if (code === 'ENOENT') { @@ -58,19 +62,29 @@ export async function recursiveDelete( // readdir does not follow symbolic links // if part is a symbolic link, follow it using stat let isDirectory = part.isDirectory() - if (part.isSymbolicLink()) { - const stats = await promises.stat(absolutePath) - isDirectory = stats.isDirectory() + const isSymlink = part.isSymbolicLink() + + if (isSymlink) { + const linkPath = await promises.readlink(absolutePath) + + try { + const stats = await promises.stat( + isAbsolute(linkPath) + ? linkPath + : join(dirname(absolutePath), linkPath) + ) + isDirectory = stats.isDirectory() + } catch (_) {} } const pp = join(previousPath, part.name) - if (isDirectory && (!exclude || !exclude.test(pp))) { - await recursiveDelete(absolutePath, exclude, pp) - return promises.rmdir(absolutePath) - } + const isNotExcluded = !exclude || !exclude.test(pp) - if (!exclude || !exclude.test(pp)) { - return unlinkFile(absolutePath) + if (isNotExcluded) { + if (isDirectory) { + await recursiveDelete(absolutePath, exclude, pp) + } + return unlinkPath(absolutePath, !isSymlink && isDirectory) } }) ) diff --git a/packages/next/server/config-shared.ts b/packages/next/server/config-shared.ts index 5ee94ee1a40105f..dcca4d5163c3e5c 100644 --- a/packages/next/server/config-shared.ts +++ b/packages/next/server/config-shared.ts @@ -156,6 +156,7 @@ export type NextConfig = { [key: string]: any } & { fullySpecified?: boolean urlImports?: NonNullable['buildHttp'] outputFileTracingRoot?: string + outputStandalone?: boolean } } @@ -238,6 +239,7 @@ export const defaultConfig: NextConfig = { serverComponents: false, fullySpecified: false, outputFileTracingRoot: process.env.NEXT_PRIVATE_OUTPUT_TRACE_ROOT || '', + outputStandalone: !!process.env.NEXT_PRIVATE_STANDALONE, }, future: { strictPostcssConfiguration: false, diff --git a/packages/next/server/next-server.ts b/packages/next/server/next-server.ts index 1d4d37651ed1448..b2d56eb2d1c480b 100644 --- a/packages/next/server/next-server.ts +++ b/packages/next/server/next-server.ts @@ -2538,11 +2538,13 @@ export default class Server { } let nextFilesStatic: string[] = [] - nextFilesStatic = !this.minimalMode - ? recursiveReadDirSync(join(this.distDir, 'static')).map((f) => - join('.', relative(this.dir, this.distDir), 'static', f) - ) - : [] + + nextFilesStatic = + !this.minimalMode && fs.existsSync(join(this.distDir, 'static')) + ? recursiveReadDirSync(join(this.distDir, 'static')).map((f) => + join('.', relative(this.dir, this.distDir), 'static', f) + ) + : [] return (this._validFilesystemPathSet = new Set([ ...nextFilesStatic, diff --git a/test/production/required-server-files.test.ts b/test/production/required-server-files.test.ts index eaf68c442333168..f7b9fafc25e2228 100644 --- a/test/production/required-server-files.test.ts +++ b/test/production/required-server-files.test.ts @@ -1,7 +1,7 @@ import glob from 'glob' -import _fs from 'fs-extra' +import fs from 'fs-extra' import cheerio from 'cheerio' -import { join, dirname } from 'path' +import { join } from 'path' import { createNext, FileRef } from 'e2e-utils' import { NextInstance } from 'test/lib/next-modes/base' import { @@ -33,6 +33,9 @@ describe('should set-up next', () => { eslint: { ignoreDuringBuilds: true, }, + experimental: { + outputStandalone: true, + }, async rewrites() { return [ { @@ -48,94 +51,47 @@ describe('should set-up next', () => { }, }) await next.stop() - const keptFiles = new Set() - const nextServerTrace = require(join( - next.testDir, - '.next/next-server.js.nft.json' - )) requiredFilesManifest = JSON.parse( await next.readFile('.next/required-server-files.json') ) - requiredFilesManifest.files.forEach((file) => keptFiles.add(file)) - - const pageTraceFiles = glob.sync('**/*.nft.json', { - cwd: join(next.testDir, '.next/server/pages'), - }) - for (const traceFile of pageTraceFiles) { - const pageDir = dirname(join('.next/server/pages', traceFile)) - const trace = await _fs.readJSON( - join(next.testDir, '.next/server/pages', traceFile) - ) - keptFiles.add( - join('.next/server/pages', traceFile.replace('.nft.json', '')) - ) - - for (const file of trace.files) { - keptFiles.add(join(pageDir, file)) + await fs.move( + join(next.testDir, '.next/standalone'), + join(next.testDir, 'standalone') + ) + for (const file of await fs.readdir(next.testDir)) { + if (file !== 'standalone') { + await fs.remove(join(next.testDir, file)) + console.log('removed', file) } } - - const allFiles = glob.sync('**/*', { - cwd: next.testDir, + const files = glob.sync('**/*', { + cwd: join(next.testDir, 'standalone/.next/server/pages'), dot: true, }) - const nextServerTraceFiles = nextServerTrace.files.map((file) => { - return join(next.testDir, '.next', file) - }) + console.error({ files }) - for (const file of allFiles) { - const filePath = join(next.testDir, file) - if ( - !keptFiles.has(file) && - !(await _fs.stat(filePath).catch(() => null))?.isDirectory() && - !nextServerTraceFiles.includes(filePath) && - !file.match(/node_modules\/(react|react-dom)\//) && - file !== 'node_modules/next/dist/server/next-server.js' - ) { - await _fs.remove(filePath) + for (const file of files) { + if (file.endsWith('.json') || file.endsWith('.html')) { + await fs.remove(join(next.testDir, '.next/server', file)) } } - appPort = await findPort() - const testServer = join(next.testDir, 'server.js') - await _fs.writeFile( + const testServer = join(next.testDir, 'standalone/server.js') + await fs.writeFile( testServer, - ` - const http = require('http') - const NextServer = require('next/dist/server/next-server').default - const appPort = ${appPort} - - const nextApp = new NextServer({ - conf: ${JSON.stringify(requiredFilesManifest.config)}, - dir: "${next.testDir}", - quiet: false, - minimalMode: true, - }) - - server = http.createServer(async (req, res) => { - try { - await nextApp.getRequestHandler()(req, res) - } catch (err) { - console.error('top-level', err) - res.statusCode = 500 - res.end('error') - } - }) - server.listen(appPort, (err) => { - if (err) throw err - console.log(\`Listening at ::${appPort}\`) - }) - ` + (await fs.readFile(testServer, 'utf8')) + .replace('console.error(err)', `console.error('top-level', err)`) + .replace('conf:', 'minimalMode: true,conf:') ) - + appPort = await findPort() server = await initNextServerScript( testServer, - /Listening at/, + /Listening on/, { ...process.env, - NODE_ENV: 'production', + PORT: appPort, }, undefined, { @@ -165,7 +121,7 @@ describe('should set-up next', () => { }) it('should set correct SWR headers with notFound gsp', async () => { - await next.patchFile('data.txt', 'show') + await next.patchFile('standalone/data.txt', 'show') const res = await fetchViaHTTP(appPort, '/gsp', undefined, { redirect: 'manual ', @@ -175,7 +131,7 @@ describe('should set-up next', () => { 's-maxage=1, stale-while-revalidate' ) - await next.patchFile('data.txt', 'hide') + await next.patchFile('standalone/data.txt', 'hide') const res2 = await fetchViaHTTP(appPort, '/gsp', undefined, { redirect: 'manual ', @@ -187,7 +143,7 @@ describe('should set-up next', () => { }) it('should set correct SWR headers with notFound gssp', async () => { - await next.patchFile('data.txt', 'show') + await next.patchFile('standalone/data.txt', 'show') const res = await fetchViaHTTP(appPort, '/gssp', undefined, { redirect: 'manual ', @@ -197,7 +153,7 @@ describe('should set-up next', () => { 's-maxage=1, stale-while-revalidate' ) - await next.patchFile('data.txt', 'hide') + await next.patchFile('standalone/data.txt', 'hide') const res2 = await fetchViaHTTP(appPort, '/gssp', undefined, { redirect: 'manual ', @@ -576,7 +532,7 @@ describe('should set-up next', () => { errors = [] const res = await fetchViaHTTP(appPort, '/errors/gip', { crash: '1' }) expect(res.status).toBe(500) - expect(await res.text()).toBe('error') + expect(await res.text()).toBe('internal server error') await check( () => (errors[0].includes('gip hit an oops') ? 'success' : errors[0]), @@ -588,7 +544,7 @@ describe('should set-up next', () => { errors = [] const res = await fetchViaHTTP(appPort, '/errors/gssp', { crash: '1' }) expect(res.status).toBe(500) - expect(await res.text()).toBe('error') + expect(await res.text()).toBe('internal server error') await check( () => (errors[0].includes('gssp hit an oops') ? 'success' : errors[0]), 'success' @@ -599,7 +555,7 @@ describe('should set-up next', () => { errors = [] const res = await fetchViaHTTP(appPort, '/errors/gsp/crash') expect(res.status).toBe(500) - expect(await res.text()).toBe('error') + expect(await res.text()).toBe('internal server error') await check( () => (errors[0].includes('gsp hit an oops') ? 'success' : errors[0]), 'success' @@ -610,7 +566,7 @@ describe('should set-up next', () => { errors = [] const res = await fetchViaHTTP(appPort, '/api/error') expect(res.status).toBe(500) - expect(await res.text()).toBe('error') + expect(await res.text()).toBe('internal server error') await check( () => errors[0].includes('some error from /api/error') diff --git a/test/unit/recursive-delete.test.ts b/test/unit/recursive-delete.test.ts index a2c4f7f43da9b80..e2b8c293f101932 100644 --- a/test/unit/recursive-delete.test.ts +++ b/test/unit/recursive-delete.test.ts @@ -1,4 +1,5 @@ /* eslint-env jest */ +import fs from 'fs-extra' import { recursiveDelete } from 'next/dist/lib/recursive-delete' import { recursiveReadDir } from 'next/dist/lib/recursive-readdir' import { recursiveCopy } from 'next/dist/lib/recursive-copy' @@ -13,6 +14,7 @@ describe('recursiveDelete', () => { expect.assertions(1) try { await recursiveCopy(resolveDataDir, testResolveDataDir) + await fs.symlink('./aa', join(testResolveDataDir, 'symlink')) await recursiveDelete(testResolveDataDir) const result = await recursiveReadDir(testResolveDataDir, /.*/) expect(result.length).toBe(0)