feat(next/swc): allow to run custom turbopack binary (#42656)

<!--
Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change that you're making:
-->

This PR exposes internal injection point to specify custom binary /
bindings for the turbopack. It doesn't affect any existing native
bindings (swc), but only allows to run custom turbopack
(`turbo.startDev`) if specified.

The changes are mostly for the internal dev / testing iteration purpose.
Currently, the workflow to test new turbopack changes in next.js is

- update next-swc's cargo to point to specific changes in turbopack
- rebuild, specify next.js to use local binary

This requires additional build times (turbopack build cannot be used,
have to rebuild next-swc napi bindings) as well as if we'd like to try
some changes on the CI between next.js canary + latest turbopack,
integration gets bit tricky. 2 env variable is exposed for the purpose -
`__INTERNAL_CUSTOM_TURBOPACK_BINARY` stands for the executable binary
(next-dev) can be built in turbopack,
`__INTERNAL_CUSTOM_TURBOPACK_BINDINGS` stands for the custom napi
bindings if needed. Both are strictly internal as name implies. There
won't be any documentation, no semver guarantee or else. Current plan is
to execute some subset of next.js test on turbopack PRs with these
entrypoints.

Feature / security wise these are mostly harmless I believe - this is
devserver usually don't run on CI, and trying to print out warning if
flag is enabled.

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm build && pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)

Co-authored-by: JJ Kasper <jj@jjsweb.site>

packages/next/build/swc/index.d.ts

@@ -9,3 +9,5 @@ export function initCustomTraceSubscriber(traceFileName?: string): void
 export function teardownTraceSubscriber(): void
 export function teardownCrashReporter(): void
 export function loadBindings(): Promise<void>
+
+export function __isCustomTurbopackBinary(): Promise<boolean>

packages/next/build/swc/index.js

@@ -13,6 +13,30 @@ const ArchName = arch()
 const PlatformName = platform()
 const triples = platformArchTriples[PlatformName][ArchName] || []
 
+// Allow to specify an absolute path to the custom turbopack binary to load.
+// If one of env variables is set, `loadNative` will try to use any turbo-* interfaces from specified
+// binary instead. This will not affect existing swc's transform, or other interfaces. This is thin,
+// naive interface - `loadBindings` will not validate neither path nor the binary.
+//
+// Note these are internal flag: there's no stability, feature gaurentee.
+const __INTERNAL_CUSTOM_TURBOPACK_BINARY =
+  process.env.__INTERNAL_CUSTOM_TURBOPACK_BINARY
+const __INTERNAL_CUSTOM_TURBOPACK_BINDINGS =
+  process.env.__INTERNAL_CUSTOM_TURBOPACK_BINDINGS
+export const __isCustomTurbopackBinary = async () => {
+  if (
+    !!__INTERNAL_CUSTOM_TURBOPACK_BINARY &&
+    !!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS
+  ) {
+    throw new Error('Cannot use TURBOPACK_BINARY and TURBOPACK_BINDINGS both')
+  }
+
+  return (
+    !!__INTERNAL_CUSTOM_TURBOPACK_BINARY ||
+    !!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS
+  )
+}
+
 // These are the platforms we'll try to load wasm bindings first,
 // only try to load native bindings if loading wasm binding somehow fails.
 // Fallback to native binding is for migration period only,
@@ -39,6 +63,7 @@ export async function loadBindings() {
   if (pendingBindings) {
     return pendingBindings
   }
+  const isCustomTurbopack = await __isCustomTurbopackBinary()
   pendingBindings = new Promise(async (resolve, reject) => {
     if (!lockfilePatchPromise.cur) {
       // always run lockfile check once so that it gets patched
@@ -62,7 +87,7 @@ export async function loadBindings() {
     }
 
     try {
-      return resolve(loadNative())
+      return resolve(loadNative(isCustomTurbopack))
     } catch (a) {
       attempts = attempts.concat(a)
     }
@@ -238,7 +263,7 @@ async function loadWasm(importPath = '') {
   throw attempts
 }
 
-function loadNative() {
+function loadNative(isCustomTurbopack = false) {
   if (nativeBindings) {
     return nativeBindings
   }
@@ -351,7 +376,60 @@ function loadNative() {
             ...options,
             noOpen: options.noOpen ?? true,
           }
-          bindings.startTurboDev(toBuffer(devOptions))
+
+          if (!isCustomTurbopack) {
+            bindings.startTurboDev(toBuffer(devOptions))
+          } else if (!!__INTERNAL_CUSTOM_TURBOPACK_BINARY) {
+            console.warn(
+              `Loading custom turbopack binary from ${__INTERNAL_CUSTOM_TURBOPACK_BINARY}`
+            )
+
+            return new Promise((resolve, reject) => {
+              const spawn = require('next/dist/compiled/cross-spawn')
+              const args = []
+
+              Object.entries(devOptions).forEach(([key, value]) => {
+                let cli_key = `--${key.replace(
+                  /[A-Z]/g,
+                  (m) => '-' + m.toLowerCase()
+                )}`
+                if (key === 'dir') {
+                  args.push(value)
+                } else if (typeof value === 'boolean' && value === true) {
+                  args.push(cli_key)
+                } else if (typeof value !== 'boolean' && !!value) {
+                  args.push(cli_key, value)
+                }
+              })
+
+              console.warn(`Running turbopack with args: [${args.join(' ')}]`)
+
+              const child = spawn(__INTERNAL_CUSTOM_TURBOPACK_BINARY, args, {
+                stdio: 'inherit',
+                env: {
+                  ...process.env,
+                },
+              })
+              child.on('close', (code) => {
+                if (code !== 0) {
+                  reject({
+                    command: `${__INTERNAL_CUSTOM_TURBOPACK_BINARY} ${args.join(
+                      ' '
+                    )}`,
+                  })
+                  return
+                }
+                resolve(0)
+              })
+            })
+          } else if (!!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS) {
+            console.warn(
+              `Loading custom turbopack bindings from ${__INTERNAL_CUSTOM_TURBOPACK_BINARY}`
+            )
+            console.warn(`Running turbopack with args: `, devOptions)
+
+            require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS).startDev(devOptions)
+          }
         },
         startTrace: (options = {}) =>
           bindings.runTurboTracing(toBuffer({ exact: true, ...options })),