From cba88fd7ffc342190e217cced66f30015edc1361 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 10 Aug 2021 19:56:50 +0200 Subject: [PATCH] use a shared worker pool for collecting page data and static page generation this avoid loading all code twice --- errors/static-page-generation-timeout.md | 3 +- packages/next/build/index.ts | 103 ++++++++++++++--------- packages/next/build/worker.ts | 2 + packages/next/export/index.ts | 61 +++++++++----- packages/next/server/config-shared.ts | 2 - 5 files changed, 104 insertions(+), 67 deletions(-) create mode 100644 packages/next/build/worker.ts diff --git a/errors/static-page-generation-timeout.md b/errors/static-page-generation-timeout.md index 604e0364870f5b2..6cef054dc5ddd0b 100644 --- a/errors/static-page-generation-timeout.md +++ b/errors/static-page-generation-timeout.md @@ -9,10 +9,11 @@ When restarted it will retry all uncompleted jobs, but if a job was unsuccessful #### Possible Ways to Fix It - Make sure that there is no infinite loop during execution. -- Make sure all Promises in `getStaticProps` `resolve` or `reject` correctly. +- Make sure all Promises in `getStaticPaths`/`getStaticProps` `resolve` or `reject` correctly. - Avoid very long timeouts for network requests. - Increase the timeout by changing the `experimental.staticPageGenerationTimeout` configuration option (default `60` in seconds). ### Useful Links +- [`getStaticPaths`](https://nextjs.org/docs/basic-features/data-fetching#getstaticpaths-static-generation) - [`getStaticProps`](https://nextjs.org/docs/basic-features/data-fetching#getstaticprops-static-generation) diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts index 6621ed8ef2cf5b1..66bdf63a91a414e 100644 --- a/packages/next/build/index.ts +++ b/packages/next/build/index.ts @@ -91,7 +91,7 @@ import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path' import { isWebpack5 } from 'next/dist/compiled/webpack/webpack' import { NextConfigComplete } from '../server/config-shared' -const staticCheckWorker = require.resolve('./utils') +const staticWorker = require.resolve('./worker') export type SsgRoute = { initialRevalidateSeconds: number | false @@ -670,6 +670,56 @@ export default async function build( await promises.readFile(buildManifestPath, 'utf8') ) as BuildManifest + const timeout = config.experimental.staticPageGenerationTimeout || 0 + let infoPrinted = false + const staticWorkers = new Worker(staticWorker, { + timeout: timeout * 1000, + onRestart: (method, [arg], attempts) => { + if (method === 'exportPage') { + const { path: pagePath } = arg + if (attempts >= 3) { + throw new Error( + `Static page generation for ${pagePath} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout` + ) + } + Log.warn( + `Restarted static page genertion for ${pagePath} because it took more than ${timeout} seconds` + ) + } else { + const pagePath = arg + if (attempts >= 2) { + throw new Error( + `Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout` + ) + } + Log.warn( + `Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds` + ) + } + if (!infoPrinted) { + Log.warn( + 'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout' + ) + infoPrinted = true + } + }, + numWorkers: config.experimental.cpus, + enableWorkerThreads: config.experimental.workerThreads, + exposedMethods: [ + 'hasCustomGetInitialProps', + 'isPageStatic', + 'getNamedExports', + 'exportPage', + ], + }) as Worker & + Pick< + typeof import('./worker'), + | 'hasCustomGetInitialProps' + | 'isPageStatic' + | 'getNamedExports' + | 'exportPage' + > + const analysisBegin = process.hrtime() const staticCheckSpan = nextBuildSpan.traceChild('static-check') @@ -682,39 +732,6 @@ export default async function build( } = await staticCheckSpan.traceAsyncFn(async () => { process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD - const timeout = config.experimental.pageDataCollectionTimeout || 0 - let infoPrinted = false - const staticCheckWorkers = new Worker(staticCheckWorker, { - timeout: timeout * 1000, - onRestart: (_method, [pagePath], attempts) => { - if (attempts >= 2) { - throw new Error( - `Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout` - ) - } - Log.warn( - `Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds` - ) - if (!infoPrinted) { - Log.warn( - 'See more info here https://nextjs.org/docs/messages/page-data-collection-timeout' - ) - infoPrinted = true - } - }, - numWorkers: config.experimental.cpus, - enableWorkerThreads: config.experimental.workerThreads, - exposedMethods: [ - 'hasCustomGetInitialProps', - 'isPageStatic', - 'getNamedExports', - ], - }) as Worker & - Pick< - typeof import('./utils'), - 'hasCustomGetInitialProps' | 'isPageStatic' | 'getNamedExports' - > - const runtimeEnvConfig = { publicRuntimeConfig: config.publicRuntimeConfig, serverRuntimeConfig: config.serverRuntimeConfig, @@ -726,7 +743,7 @@ export default async function build( const errorPageHasCustomGetInitialProps = nonStaticErrorPageSpan.traceAsyncFn( async () => hasCustomErrorPage && - (await staticCheckWorkers.hasCustomGetInitialProps( + (await staticWorkers.hasCustomGetInitialProps( '/_error', distDir, isLikeServerless, @@ -738,7 +755,7 @@ export default async function build( const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn( async () => hasCustomErrorPage && - staticCheckWorkers.isPageStatic( + staticWorkers.isPageStatic( '/_error', distDir, isLikeServerless, @@ -753,7 +770,7 @@ export default async function build( // from _error instead const appPageToCheck = isLikeServerless ? '/_error' : '/_app' - const customAppGetInitialPropsPromise = staticCheckWorkers.hasCustomGetInitialProps( + const customAppGetInitialPropsPromise = staticWorkers.hasCustomGetInitialProps( appPageToCheck, distDir, isLikeServerless, @@ -761,7 +778,7 @@ export default async function build( true ) - const namedExportsPromise = staticCheckWorkers.getNamedExports( + const namedExportsPromise = staticWorkers.getNamedExports( appPageToCheck, distDir, isLikeServerless, @@ -808,7 +825,7 @@ export default async function build( 'is-page-static' ) let workerResult = await isPageStaticSpan.traceAsyncFn(() => { - return staticCheckWorkers.isPageStatic( + return staticWorkers.isPageStatic( page, distDir, isLikeServerless, @@ -926,7 +943,6 @@ export default async function build( hasNonStaticErrorPage: nonStaticErrorPage, } - staticCheckWorkers.end() return returnValue }) @@ -1082,7 +1098,8 @@ export default async function build( ssgPages, additionalSsgPaths ) - const exportApp = require('../export').default + const exportApp: typeof import('../export').default = require('../export') + .default const exportOptions = { silent: false, buildExport: true, @@ -1090,6 +1107,10 @@ export default async function build( pages: combinedPages, outdir: path.join(distDir, 'export'), statusMessage: 'Generating static pages', + exportPageWorker: staticWorkers.exportPage.bind(staticWorkers), + endWorker: async () => { + await staticWorkers.end() + }, } const exportConfig: any = { ...config, diff --git a/packages/next/build/worker.ts b/packages/next/build/worker.ts new file mode 100644 index 000000000000000..fe111eda503d3aa --- /dev/null +++ b/packages/next/build/worker.ts @@ -0,0 +1,2 @@ +export * from './utils' +export { default as exportPage } from '../export/worker' diff --git a/packages/next/export/index.ts b/packages/next/export/index.ts index 3ed7dd6fed3b926..727037872228910 100644 --- a/packages/next/export/index.ts +++ b/packages/next/export/index.ts @@ -137,6 +137,8 @@ interface ExportOptions { pages?: string[] buildExport?: boolean statusMessage?: string + exportPageWorker?: typeof import('./worker').default + endWorker?: () => Promise } export default async function exportApp( @@ -519,29 +521,40 @@ export default async function exportApp( const timeout = configuration?.experimental.staticPageGenerationTimeout || 0 let infoPrinted = false - const worker = new Worker(require.resolve('./worker'), { - timeout: timeout * 1000, - onRestart: (_method, [{ path }], attempts) => { - if (attempts >= 3) { - throw new Error( - `Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout` - ) - } - Log.warn( - `Restarted static page genertion for ${path} because it took more than ${timeout} seconds` - ) - if (!infoPrinted) { + let exportPage: typeof import('./worker').default + let endWorker: () => Promise + if (options.exportPageWorker) { + exportPage = options.exportPageWorker + endWorker = options.endWorker || (() => Promise.resolve()) + } else { + const worker = new Worker(require.resolve('./worker'), { + timeout: timeout * 1000, + onRestart: (_method, [{ path }], attempts) => { + if (attempts >= 3) { + throw new Error( + `Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout` + ) + } Log.warn( - 'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout' + `Restarted static page genertion for ${path} because it took more than ${timeout} seconds` ) - infoPrinted = true - } - }, - maxRetries: 0, - numWorkers: threads, - enableWorkerThreads: nextConfig.experimental.workerThreads, - exposedMethods: ['default'], - }) as Worker & typeof import('./worker') + if (!infoPrinted) { + Log.warn( + 'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout' + ) + infoPrinted = true + } + }, + maxRetries: 0, + numWorkers: threads, + enableWorkerThreads: nextConfig.experimental.workerThreads, + exposedMethods: ['default'], + }) as Worker & typeof import('./worker') + exportPage = worker.default.bind(worker) + endWorker = async () => { + await worker.end() + } + } let renderError = false const errorPaths: string[] = [] @@ -553,7 +566,7 @@ export default async function exportApp( return pageExportSpan.traceAsyncFn(async () => { const pathMap = exportPathMap[path] - const result = await worker.default({ + const result = await exportPage({ path, pathMap, distDir, @@ -604,7 +617,7 @@ export default async function exportApp( }) ) - worker.end() + const endWorkerPromise = endWorker() // copy prerendered routes to outDir if (!options.buildExport && prerenderManifest) { @@ -681,5 +694,7 @@ export default async function exportApp( if (telemetry) { await telemetry.flush() } + + await endWorkerPromise }) } diff --git a/packages/next/server/config-shared.ts b/packages/next/server/config-shared.ts index f8a7e7f41888cdd..a995e2d353e6d3d 100644 --- a/packages/next/server/config-shared.ts +++ b/packages/next/server/config-shared.ts @@ -111,7 +111,6 @@ export type NextConfig = { [key: string]: any } & { craCompat?: boolean esmExternals?: boolean | 'loose' staticPageGenerationTimeout?: number - pageDataCollectionTimeout?: number isrMemoryCacheSize?: number } } @@ -182,7 +181,6 @@ export const defaultConfig: NextConfig = { craCompat: false, esmExternals: false, staticPageGenerationTimeout: 60, - pageDataCollectionTimeout: 60, // default to 50MB limit isrMemoryCacheSize: 50 * 1024 * 1024, },