use a shared worker pool for collecting page data and static page gen…

…eration this avoid loading all code twice
vercel · Aug 10, 2021 · cba88fd · cba88fd
1 parent 43393d5
commit cba88fd
Show file tree

Hide file tree

Showing 5 changed files with 104 additions and 67 deletions.
diff --git a/errors/static-page-generation-timeout.md b/errors/static-page-generation-timeout.md
@@ -9,10 +9,11 @@ When restarted it will retry all uncompleted jobs, but if a job was unsuccessful
 #### Possible Ways to Fix It
 
 - Make sure that there is no infinite loop during execution.
-- Make sure all Promises in `getStaticProps` `resolve` or `reject` correctly.
+- Make sure all Promises in `getStaticPaths`/`getStaticProps` `resolve` or `reject` correctly.
 - Avoid very long timeouts for network requests.
 - Increase the timeout by changing the `experimental.staticPageGenerationTimeout` configuration option (default `60` in seconds).
 
 ### Useful Links
 
+- [`getStaticPaths`](https://nextjs.org/docs/basic-features/data-fetching#getstaticpaths-static-generation)
 - [`getStaticProps`](https://nextjs.org/docs/basic-features/data-fetching#getstaticprops-static-generation)
diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts
@@ -91,7 +91,7 @@ import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
 import { isWebpack5 } from 'next/dist/compiled/webpack/webpack'
 import { NextConfigComplete } from '../server/config-shared'
 
-const staticCheckWorker = require.resolve('./utils')
+const staticWorker = require.resolve('./worker')
 
 export type SsgRoute = {
   initialRevalidateSeconds: number | false
@@ -670,6 +670,56 @@ export default async function build(
       await promises.readFile(buildManifestPath, 'utf8')
     ) as BuildManifest
 
+    const timeout = config.experimental.staticPageGenerationTimeout || 0
+    let infoPrinted = false
+    const staticWorkers = new Worker(staticWorker, {
+      timeout: timeout * 1000,
+      onRestart: (method, [arg], attempts) => {
+        if (method === 'exportPage') {
+          const { path: pagePath } = arg
+          if (attempts >= 3) {
+            throw new Error(
+              `Static page generation for ${pagePath} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
+            )
+          }
+          Log.warn(
+            `Restarted static page genertion for ${pagePath} because it took more than ${timeout} seconds`
+          )
+        } else {
+          const pagePath = arg
+          if (attempts >= 2) {
+            throw new Error(
+              `Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout`
+            )
+          }
+          Log.warn(
+            `Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds`
+          )
+        }
+        if (!infoPrinted) {
+          Log.warn(
+            'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
+          )
+          infoPrinted = true
+        }
+      },
+      numWorkers: config.experimental.cpus,
+      enableWorkerThreads: config.experimental.workerThreads,
+      exposedMethods: [
+        'hasCustomGetInitialProps',
+        'isPageStatic',
+        'getNamedExports',
+        'exportPage',
+      ],
+    }) as Worker &
+      Pick<
+        typeof import('./worker'),
+        | 'hasCustomGetInitialProps'
+        | 'isPageStatic'
+        | 'getNamedExports'
+        | 'exportPage'
+      >
+
     const analysisBegin = process.hrtime()
 
     const staticCheckSpan = nextBuildSpan.traceChild('static-check')
@@ -682,39 +732,6 @@ export default async function build(
     } = await staticCheckSpan.traceAsyncFn(async () => {
       process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
 
-      const timeout = config.experimental.pageDataCollectionTimeout || 0
-      let infoPrinted = false
-      const staticCheckWorkers = new Worker(staticCheckWorker, {
-        timeout: timeout * 1000,
-        onRestart: (_method, [pagePath], attempts) => {
-          if (attempts >= 2) {
-            throw new Error(
-              `Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout`
-            )
-          }
-          Log.warn(
-            `Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds`
-          )
-          if (!infoPrinted) {
-            Log.warn(
-              'See more info here https://nextjs.org/docs/messages/page-data-collection-timeout'
-            )
-            infoPrinted = true
-          }
-        },
-        numWorkers: config.experimental.cpus,
-        enableWorkerThreads: config.experimental.workerThreads,
-        exposedMethods: [
-          'hasCustomGetInitialProps',
-          'isPageStatic',
-          'getNamedExports',
-        ],
-      }) as Worker &
-        Pick<
-          typeof import('./utils'),
-          'hasCustomGetInitialProps' | 'isPageStatic' | 'getNamedExports'
-        >
-
       const runtimeEnvConfig = {
         publicRuntimeConfig: config.publicRuntimeConfig,
         serverRuntimeConfig: config.serverRuntimeConfig,
@@ -726,7 +743,7 @@ export default async function build(
       const errorPageHasCustomGetInitialProps = nonStaticErrorPageSpan.traceAsyncFn(
         async () =>
           hasCustomErrorPage &&
-          (await staticCheckWorkers.hasCustomGetInitialProps(
+          (await staticWorkers.hasCustomGetInitialProps(
             '/_error',
             distDir,
             isLikeServerless,
@@ -738,7 +755,7 @@ export default async function build(
       const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn(
         async () =>
           hasCustomErrorPage &&
-          staticCheckWorkers.isPageStatic(
+          staticWorkers.isPageStatic(
             '/_error',
             distDir,
             isLikeServerless,
@@ -753,15 +770,15 @@ export default async function build(
       // from _error instead
       const appPageToCheck = isLikeServerless ? '/_error' : '/_app'
 
-      const customAppGetInitialPropsPromise = staticCheckWorkers.hasCustomGetInitialProps(
+      const customAppGetInitialPropsPromise = staticWorkers.hasCustomGetInitialProps(
         appPageToCheck,
         distDir,
         isLikeServerless,
         runtimeEnvConfig,
         true
       )
 
-      const namedExportsPromise = staticCheckWorkers.getNamedExports(
+      const namedExportsPromise = staticWorkers.getNamedExports(
         appPageToCheck,
         distDir,
         isLikeServerless,
@@ -808,7 +825,7 @@ export default async function build(
                   'is-page-static'
                 )
                 let workerResult = await isPageStaticSpan.traceAsyncFn(() => {
-                  return staticCheckWorkers.isPageStatic(
+                  return staticWorkers.isPageStatic(
                     page,
                     distDir,
                     isLikeServerless,
@@ -926,7 +943,6 @@ export default async function build(
         hasNonStaticErrorPage: nonStaticErrorPage,
       }
 
-      staticCheckWorkers.end()
       return returnValue
     })
 
@@ -1082,14 +1098,19 @@ export default async function build(
           ssgPages,
           additionalSsgPaths
         )
-        const exportApp = require('../export').default
+        const exportApp: typeof import('../export').default = require('../export')
+          .default
         const exportOptions = {
           silent: false,
           buildExport: true,
           threads: config.experimental.cpus,
           pages: combinedPages,
           outdir: path.join(distDir, 'export'),
           statusMessage: 'Generating static pages',
+          exportPageWorker: staticWorkers.exportPage.bind(staticWorkers),
+          endWorker: async () => {
+            await staticWorkers.end()
+          },
         }
         const exportConfig: any = {
           ...config,

diff --git a/packages/next/build/worker.ts b/packages/next/build/worker.ts
@@ -0,0 +1,2 @@
+export * from './utils'
+export { default as exportPage } from '../export/worker'
diff --git a/packages/next/export/index.ts b/packages/next/export/index.ts
@@ -137,6 +137,8 @@ interface ExportOptions {
   pages?: string[]
   buildExport?: boolean
   statusMessage?: string
+  exportPageWorker?: typeof import('./worker').default
+  endWorker?: () => Promise<void>
 }
 
 export default async function exportApp(
@@ -519,29 +521,40 @@ export default async function exportApp(
 
     const timeout = configuration?.experimental.staticPageGenerationTimeout || 0
     let infoPrinted = false
-    const worker = new Worker(require.resolve('./worker'), {
-      timeout: timeout * 1000,
-      onRestart: (_method, [{ path }], attempts) => {
-        if (attempts >= 3) {
-          throw new Error(
-            `Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
-          )
-        }
-        Log.warn(
-          `Restarted static page genertion for ${path} because it took more than ${timeout} seconds`
-        )
-        if (!infoPrinted) {
+    let exportPage: typeof import('./worker').default
+    let endWorker: () => Promise<void>
+    if (options.exportPageWorker) {
+      exportPage = options.exportPageWorker
+      endWorker = options.endWorker || (() => Promise.resolve())
+    } else {
+      const worker = new Worker(require.resolve('./worker'), {
+        timeout: timeout * 1000,
+        onRestart: (_method, [{ path }], attempts) => {
+          if (attempts >= 3) {
+            throw new Error(
+              `Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
+            )
+          }
           Log.warn(
-            'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
+            `Restarted static page genertion for ${path} because it took more than ${timeout} seconds`
           )
-          infoPrinted = true
-        }
-      },
-      maxRetries: 0,
-      numWorkers: threads,
-      enableWorkerThreads: nextConfig.experimental.workerThreads,
-      exposedMethods: ['default'],
-    }) as Worker & typeof import('./worker')
+          if (!infoPrinted) {
+            Log.warn(
+              'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
+            )
+            infoPrinted = true
+          }
+        },
+        maxRetries: 0,
+        numWorkers: threads,
+        enableWorkerThreads: nextConfig.experimental.workerThreads,
+        exposedMethods: ['default'],
+      }) as Worker & typeof import('./worker')
+      exportPage = worker.default.bind(worker)
+      endWorker = async () => {
+        await worker.end()
+      }
+    }
 
     let renderError = false
     const errorPaths: string[] = []
@@ -553,7 +566,7 @@ export default async function exportApp(
 
         return pageExportSpan.traceAsyncFn(async () => {
           const pathMap = exportPathMap[path]
-          const result = await worker.default({
+          const result = await exportPage({
             path,
             pathMap,
             distDir,
@@ -604,7 +617,7 @@ export default async function exportApp(
       })
     )
 
-    worker.end()
+    const endWorkerPromise = endWorker()
 
     // copy prerendered routes to outDir
     if (!options.buildExport && prerenderManifest) {
@@ -681,5 +694,7 @@ export default async function exportApp(
     if (telemetry) {
       await telemetry.flush()
     }
+
+    await endWorkerPromise
   })
 }
diff --git a/packages/next/server/config-shared.ts b/packages/next/server/config-shared.ts
@@ -111,7 +111,6 @@ export type NextConfig = { [key: string]: any } & {
     craCompat?: boolean
     esmExternals?: boolean | 'loose'
     staticPageGenerationTimeout?: number
-    pageDataCollectionTimeout?: number
     isrMemoryCacheSize?: number
   }
 }
@@ -182,7 +181,6 @@ export const defaultConfig: NextConfig = {
     craCompat: false,
     esmExternals: false,
     staticPageGenerationTimeout: 60,
-    pageDataCollectionTimeout: 60,
     // default to 50MB limit
     isrMemoryCacheSize: 50 * 1024 * 1024,
   },