Rest of options in experimental.turbotrace and documentation

docs/advanced-features/output-file-tracing.md

@@ -66,3 +66,45 @@ module.exports = {
 
 - There are some cases in which Next.js might fail to include required files, or might incorrectly include unused files. In those cases, you can export page configs props `unstable_includeFiles` and `unstable_excludeFiles` respectively. Each prop accepts an array of [minimatch globs](https://www.npmjs.com/package/minimatch) relative to the project's root to either include or exclude in the trace.
 - Currently, Next.js does not do anything with the emitted `.nft.json` files. The files must be read by your deployment platform, for example [Vercel](https://vercel.com), to create a minimal deployment. In a future release, a new command is planned to utilize these `.nft.json` files.
+
+## Experimental `turbotrace`
+
+Tracing dependencies can be slow because it requires very complex computations and analysis. We created `turbotrace` in Rust as a faster and smarter alternative to the JavaScript implementation.
+
+To enable it, you can add the following configuration to your `next.config.js`:
+
+```js
+// next.config.js
+module.exports = {
+  experimental: {
+    turbotrace: {
+      // control the log level of the turbotrace, default is `error`
+      logLevel?:
+      | 'bug'
+      | 'fatal'
+      | 'error'
+      | 'warning'
+      | 'hint'
+      | 'note'
+      | 'suggestions'
+      | 'info',
+      // control if the log of turbotrace should contain the details of the analysis, default is `false`
+      logDetail?: boolean
+      // show all log messages without limit
+      // turbotrace only show 1 log message for each categories by default
+      logAll?: boolean
+      // control the context directory of the turbotrace
+      // files outside of the context directory will not be traced
+      // set the `experimental.outputFileTracingRoot` has the same effect
+      // if the `experimental.outputFileTracingRoot` and this option are both set, the `experimental.turbotrace.contextDirectory` will be used
+      contextDirectory?: string
+      // if there is `process.cwd()` expression in your code, you can set this option to tell `turbotrace` the value of `process.cwd()` while tracing.
+      // for example the require(process.cwd() + '/package.json') will be traced as require('/path/to/cwd/package.json')
+      processCwd?: string,
+      // control the maximum number of files that are passed to the `turbotrace`
+      // default is 128
+      maxFiles?: number;
+    },
+  },
+}
+```

packages/next-swc/crates/napi/src/lib.rs

@@ -48,8 +48,8 @@ use swc_core::{
 pub mod minify;
 pub mod parse;
 pub mod transform;
-pub mod turbo_tracing;
 pub mod turbopack;
+pub mod turbotrace;
 pub mod util;
 
 static COMPILER: Lazy<Arc<Compiler>> = Lazy::new(|| {

packages/next/build/index.ts

@@ -1751,7 +1751,10 @@ export default async function build(
               } catch (_) {}
             }
 
-            const root = config.experimental.outputFileTracingRoot || dir
+            const root =
+              config.experimental?.turbotrace?.contextDirectory ??
+              config.experimental?.outputFileTracingRoot ??
+              dir
             const toTrace = [require.resolve('next/dist/server/next-server')]
 
             // ensure we trace any dependencies needed for custom
@@ -1769,6 +1772,10 @@ export default async function build(
                 action: 'annotate',
                 input: toTrace,
                 contextDirectory: root,
+                logLevel: config.experimental.turbotrace.logLevel,
+                processCwd: config.experimental.turbotrace.processCwd,
+                logDetail: config.experimental.turbotrace.logDetail,
+                showAll: config.experimental.turbotrace.logAll,
               })
             } else {
               serverResult = await nodeFileTrace(toTrace, {

packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts

@@ -23,6 +23,8 @@ const TRACE_IGNORES = [
   '**/*/next/dist/bin/next',
 ]
 
+const TURBO_TRACE_DEFAULT_MAX_FILES = 128
+
 function getModuleFromDependency(
   compilation: any,
   dep: any
@@ -390,13 +392,33 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
                   .traceAsyncFn(async () => {
                     const contextDirectory =
                       this.turbotrace?.contextDirectory ?? this.tracingRoot
-                    const filesTracedInEntries: string[] =
-                      await binding.turbo.startTrace({
-                        action: 'print',
-                        input: entriesToTrace,
-                        contextDirectory,
-                        processCwd: this.turbotrace?.processCwd ?? this.appDir,
-                      })
+                    const maxFiles =
+                      this.turbotrace?.maxFiles ?? TURBO_TRACE_DEFAULT_MAX_FILES
+                    let chunks = [...entriesToTrace]
+                    let restChunks =
+                      chunks.length > maxFiles ? chunks.splice(maxFiles) : []
+                    let filesTracedInEntries: string[] = []
+                    while (chunks.length) {
+                      filesTracedInEntries = filesTracedInEntries.concat(
+                        await binding.turbo.startTrace({
+                          action: 'print',
+                          input: chunks,
+                          contextDirectory,
+                          processCwd:
+                            this.turbotrace?.processCwd ?? this.appDir,
+                          logLevel: this.turbotrace?.logLevel,
+                          showAll: this.turbotrace?.logAll,
+                        })
+                      )
+                      chunks = restChunks
+                      if (restChunks.length) {
+                        restChunks =
+                          chunks.length > maxFiles
+                            ? chunks.splice(maxFiles)
+                            : []
+                      }
+                    }
+
                     // only trace the assets under the appDir
                     // exclude files from node_modules, entries and processed by webpack
                     const filesTracedFromEntries = filesTracedInEntries
@@ -763,13 +785,27 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
             !binding?.isWasm &&
             typeof binding.turbo.startTrace === 'function'
           ) {
-            await binding.turbo.startTrace({
-              action: 'annotate',
-              input: this.chunksToTrace,
-              contextDirectory:
-                this.turbotrace?.contextDirectory ?? this.tracingRoot,
-              processCwd: this.turbotrace?.processCwd ?? this.appDir,
-            })
+            const maxFiles =
+              this.turbotrace?.maxFiles ?? TURBO_TRACE_DEFAULT_MAX_FILES
+            let chunks = [...this.chunksToTrace]
+            let restChunks =
+              chunks.length > maxFiles ? chunks.splice(maxFiles) : []
+            while (chunks.length) {
+              await binding.turbo.startTrace({
+                action: 'annotate',
+                input: chunks,
+                contextDirectory:
+                  this.turbotrace?.contextDirectory ?? this.tracingRoot,
+                processCwd: this.turbotrace?.processCwd ?? this.appDir,
+                showAll: this.turbotrace?.logAll,
+                logLevel: this.turbotrace?.logLevel,
+              })
+              chunks = restChunks
+              if (restChunks.length) {
+                restChunks =
+                  chunks.length > maxFiles ? chunks.splice(maxFiles) : []
+              }
+            }
             if (this.turbotraceOutputPath && this.turbotraceFiles) {
               const existedNftFile = await nodeFs.promises
                 .readFile(this.turbotraceOutputPath, 'utf8')

packages/next/server/config-schema.ts

@@ -448,6 +448,9 @@ const configSchema = {
                 'info',
               ],
             } as any,
+            logAll: {
+              type: 'boolean',
+            },
             logDetail: {
               type: 'boolean',
             },
@@ -457,6 +460,9 @@ const configSchema = {
             processCwd: {
               type: 'string',
             },
+            maxFiles: {
+              type: 'integer',
+            },
           },
         },
       },

packages/next/server/config-shared.ts

@@ -176,8 +176,10 @@ export interface ExperimentalConfig {
       | 'suggestions'
       | 'info'
     logDetail?: boolean
+    logAll?: boolean
     contextDirectory?: string
     processCwd?: string
+    maxFiles?: number
   }
 }