Rest of options in experimental.turbotrace and documentation (#41817)

<!--
Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change that you're making:
-->

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)

docs/advanced-features/output-file-tracing.md

@@ -66,3 +66,45 @@ module.exports = {
 
 - There are some cases in which Next.js might fail to include required files, or might incorrectly include unused files. In those cases, you can export page configs props `unstable_includeFiles` and `unstable_excludeFiles` respectively. Each prop accepts an array of [minimatch globs](https://www.npmjs.com/package/minimatch) relative to the project's root to either include or exclude in the trace.
 - Currently, Next.js does not do anything with the emitted `.nft.json` files. The files must be read by your deployment platform, for example [Vercel](https://vercel.com), to create a minimal deployment. In a future release, a new command is planned to utilize these `.nft.json` files.
+
+## Experimental `turbotrace`
+
+Tracing dependencies can be slow because it requires very complex computations and analysis. We created `turbotrace` in Rust as a faster and smarter alternative to the JavaScript implementation.
+
+To enable it, you can add the following configuration to your `next.config.js`:
+
+```js
+// next.config.js
+module.exports = {
+  experimental: {
+    turbotrace: {
+      // control the log level of the turbotrace, default is `error`
+      logLevel?:
+      | 'bug'
+      | 'fatal'
+      | 'error'
+      | 'warning'
+      | 'hint'
+      | 'note'
+      | 'suggestions'
+      | 'info',
+      // control if the log of turbotrace should contain the details of the analysis, default is `false`
+      logDetail?: boolean
+      // show all log messages without limit
+      // turbotrace only show 1 log message for each categories by default
+      logAll?: boolean
+      // control the context directory of the turbotrace
+      // files outside of the context directory will not be traced
+      // set the `experimental.outputFileTracingRoot` has the same effect
+      // if the `experimental.outputFileTracingRoot` and this option are both set, the `experimental.turbotrace.contextDirectory` will be used
+      contextDirectory?: string
+      // if there is `process.cwd()` expression in your code, you can set this option to tell `turbotrace` the value of `process.cwd()` while tracing.
+      // for example the require(process.cwd() + '/package.json') will be traced as require('/path/to/cwd/package.json')
+      processCwd?: string,
+      // control the maximum number of files that are passed to the `turbotrace`
+      // default is 128
+      maxFiles?: number;
+    },
+  },
+}
+```

packages/next-swc/crates/napi/src/lib.rs

@@ -48,8 +48,8 @@ use swc_core::{
 pub mod minify;
 pub mod parse;
 pub mod transform;
-pub mod turbo_tracing;
 pub mod turbopack;
+pub mod turbotrace;
 pub mod util;
 
 static COMPILER: Lazy<Arc<Compiler>> = Lazy::new(|| {

packages/next/build/index.ts

@@ -1751,7 +1751,10 @@ export default async function build(
               } catch (_) {}
             }
 
-            const root = config.experimental.outputFileTracingRoot || dir
+            const root =
+              config.experimental?.turbotrace?.contextDirectory ??
+              config.experimental?.outputFileTracingRoot ??
+              dir
             const toTrace = [require.resolve('next/dist/server/next-server')]
 
             // ensure we trace any dependencies needed for custom
@@ -1769,6 +1772,10 @@ export default async function build(
                 action: 'annotate',
                 input: toTrace,
                 contextDirectory: root,
+                logLevel: config.experimental.turbotrace.logLevel,
+                processCwd: config.experimental.turbotrace.processCwd,
+                logDetail: config.experimental.turbotrace.logDetail,
+                showAll: config.experimental.turbotrace.logAll,
               })
             } else {
               serverResult = await nodeFileTrace(toTrace, {

packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts

@@ -23,6 +23,8 @@ const TRACE_IGNORES = [
   '**/*/next/dist/bin/next',
 ]
 
+const TURBO_TRACE_DEFAULT_MAX_FILES = 128
+
 function getModuleFromDependency(
   compilation: any,
   dep: any
@@ -390,13 +392,33 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
                   .traceAsyncFn(async () => {
                     const contextDirectory =
                       this.turbotrace?.contextDirectory ?? this.tracingRoot
-                    const filesTracedInEntries: string[] =
-                      await binding.turbo.startTrace({
-                        action: 'print',
-                        input: entriesToTrace,
-                        contextDirectory,
-                        processCwd: this.turbotrace?.processCwd ?? this.appDir,
-                      })
+                    const maxFiles =
+                      this.turbotrace?.maxFiles ?? TURBO_TRACE_DEFAULT_MAX_FILES
+                    let chunks = [...entriesToTrace]
+                    let restChunks =
+                      chunks.length > maxFiles ? chunks.splice(maxFiles) : []
+                    let filesTracedInEntries: string[] = []
+                    while (chunks.length) {
+                      filesTracedInEntries = filesTracedInEntries.concat(
+                        await binding.turbo.startTrace({
+                          action: 'print',
+                          input: chunks,
+                          contextDirectory,
+                          processCwd:
+                            this.turbotrace?.processCwd ?? this.appDir,
+                          logLevel: this.turbotrace?.logLevel,
+                          showAll: this.turbotrace?.logAll,
+                        })
+                      )
+                      chunks = restChunks
+                      if (restChunks.length) {
+                        restChunks =
+                          chunks.length > maxFiles
+                            ? chunks.splice(maxFiles)
+                            : []
+                      }
+                    }
+
                     // only trace the assets under the appDir
                     // exclude files from node_modules, entries and processed by webpack
                     const filesTracedFromEntries = filesTracedInEntries
@@ -763,13 +785,27 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
             !binding?.isWasm &&
             typeof binding.turbo.startTrace === 'function'
           ) {
-            await binding.turbo.startTrace({
-              action: 'annotate',
-              input: this.chunksToTrace,
-              contextDirectory:
-                this.turbotrace?.contextDirectory ?? this.tracingRoot,
-              processCwd: this.turbotrace?.processCwd ?? this.appDir,
-            })
+            const maxFiles =
+              this.turbotrace?.maxFiles ?? TURBO_TRACE_DEFAULT_MAX_FILES
+            let chunks = [...this.chunksToTrace]
+            let restChunks =
+              chunks.length > maxFiles ? chunks.splice(maxFiles) : []
+            while (chunks.length) {
+              await binding.turbo.startTrace({
+                action: 'annotate',
+                input: chunks,
+                contextDirectory:
+                  this.turbotrace?.contextDirectory ?? this.tracingRoot,
+                processCwd: this.turbotrace?.processCwd ?? this.appDir,
+                showAll: this.turbotrace?.logAll,
+                logLevel: this.turbotrace?.logLevel,
+              })
+              chunks = restChunks
+              if (restChunks.length) {
+                restChunks =
+                  chunks.length > maxFiles ? chunks.splice(maxFiles) : []
+              }
+            }
             if (this.turbotraceOutputPath && this.turbotraceFiles) {
               const existedNftFile = await nodeFs.promises
                 .readFile(this.turbotraceOutputPath, 'utf8')

packages/next/server/config-schema.ts

@@ -448,6 +448,9 @@ const configSchema = {
                 'info',
               ],
             } as any,
+            logAll: {
+              type: 'boolean',
+            },
             logDetail: {
               type: 'boolean',
             },
@@ -457,6 +460,9 @@ const configSchema = {
             processCwd: {
               type: 'string',
             },
+            maxFiles: {
+              type: 'integer',
+            },
           },
         },
       },

packages/next/server/config-shared.ts

@@ -176,8 +176,10 @@ export interface ExperimentalConfig {
       | 'suggestions'
       | 'info'
     logDetail?: boolean
+    logAll?: boolean
     contextDirectory?: string
     processCwd?: string
+    maxFiles?: number
   }
 }