Chore/load bindings improvements (#32191)

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

.github/workflows/build_test_deploy.yml

@@ -715,6 +715,50 @@ jobs:
       - run: cd packages/next-swc && cargo test
         if: ${{ steps.docs-change.outputs.DOCS_CHANGE != 'docs only change' }}
 
+  test-wasm:
+    name: Test the wasm build
+    runs-on: ubuntu-18.04
+    needs: [build, build-native-dev, build-wasm-dev]
+
+    steps:
+      - uses: actions/cache@v2
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}
+
+      - uses: actions/download-artifact@v2
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        with:
+          name: wasm-dev-binary
+          path: packages/next-swc/crates/wasm/pkg-nodejs
+
+      - run: ls packages/next-swc/crates/wasm
+
+      - uses: actions/download-artifact@v2
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        with:
+          name: next-swc-dev-binary
+          path: packages/next-swc/native
+
+      # node version needs to be 16+ to use --no-addons option
+      - name: Setup node
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        uses: actions/setup-node@v2
+        with:
+          node-version: 16
+          check-latest: true
+
+      - run: npm i -g playwright-chromium@1.14.1 && npx playwright install-deps
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+      - run: node ./scripts/setup-wasm.mjs
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+      - run: TEST_WASM=true xvfb-run node run-tests.js test/integration/production/test/index.test.js
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
   # Build binaries for publishing
   build-native:
     needs: build
@@ -1255,4 +1299,64 @@ jobs:
           name: wasm-binaries
           path: packages/next-swc/crates/wasm/pkg-*
 
-      - run: ls packages/next-swc/crates/wasm
+  build-wasm-dev:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/cache@v2
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}-${{ github.run_number }}-${{ github.run_attempt }}
+
+      - name: Setup node
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        uses: actions/setup-node@v2
+        with:
+          node-version: 14
+
+      - name: Install Rust
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2021-11-15
+          override: true
+          target: wasm32-unknown-unknown
+
+      - name: Cache
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/
+            **/target/
+          key: ${{ runner.os }}-publish-integration
+
+      - name: Cache wasm binary
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        id: binary-cache
+        uses: actions/cache@v2
+        with:
+          path: packages/next-swc/crates/wasm/pkg-nodejs
+          key: dev-wasm-next-swc-nightly-2021-11-15-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
+
+      - name: Install wasm-pack
+        if: ${{needs.build.outputs.docsChange != 'docs only change' && steps.binary-cache.outputs.cache-hit != 'true'}}
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+
+      - name: Build
+        if: ${{needs.build.outputs.docsChange != 'docs only change' && steps.binary-cache.outputs.cache-hit != 'true'}}
+        run: (wasm-pack build packages/next-swc/crates/wasm --dev --scope=next --target nodejs)
+
+      - name: Add target to folder name
+        if: ${{needs.build.outputs.docsChange != 'docs only change' && steps.binary-cache.outputs.cache-hit != 'true'}}
+        run: mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-nodejs
+
+      - name: Upload artifact
+        if: ${{needs.build.outputs.docsChange != 'docs only change' && steps.binary-cache.outputs.cache-hit != 'true'}}
+        uses: actions/upload-artifact@v2
+        with:
+          name: wasm-dev-binary
+          path: packages/next-swc/crates/wasm/pkg-nodejs

packages/next/build/swc/index.js

@@ -6,19 +6,63 @@ const ArchName = arch()
 const PlatformName = platform()
 const triples = platformArchTriples[PlatformName][ArchName] || []
 
+let nativeBindings
+let wasmBindings
+
 async function loadBindings() {
-  return (await loadWasm()) || loadNative()
+  let attempts = []
+  try {
+    return loadNative()
+  } catch (a) {
+    attempts = attempts.concat(a)
+  }
+
+  try {
+    let bindings = await loadWasm()
+    return bindings
+  } catch (a) {
+    attempts = attempts.concat(a)
+  }
+
+  logLoadFailure(attempts)
+}
+
+function loadBindingsSync() {
+  let attempts = []
+  try {
+    return loadNative()
+  } catch (a) {
+    attempts = attempts.concat(a)
+  }
+
+  logLoadFailure(attempts)
+}
+
+function logLoadFailure(attempts) {
+  for (let attempt of attempts) {
+    Log.info(attempt)
+  }
+
+  Log.error(
+    `Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc`
+  )
+  process.exit(1)
 }
 
 async function loadWasm() {
-  // Try to load wasm bindings
-  for (let specifier of ['@next/swc-wasm-web', '@next/swc-wasm-nodejs']) {
+  if (wasmBindings) {
+    return wasmBindings
+  }
+
+  let attempts = []
+  for (let pkg of ['@next/swc-wasm-nodejs', '@next/swc-wasm-web']) {
     try {
-      let bindings = await import(specifier)
-      if (specifier === '@next/swc-wasm-web') {
+      let bindings = await import(pkg)
+      if (pkg === '@next/swc-wasm-web') {
         bindings = await bindings.default()
       }
-      return {
+      Log.info('Using experimental wasm build of next-swc')
+      wasmBindings = {
         isWasm: true,
         transform(src, options) {
           return Promise.resolve(
@@ -29,41 +73,58 @@ async function loadWasm() {
           return Promise.resolve(bindings.minifySync(src.toString(), options))
         },
       }
-    } catch (e) {}
+      return wasmBindings
+    } catch (e) {
+      // Do not report attempts to load wasm when it is still experimental
+      // if (e?.code === 'ERR_MODULE_NOT_FOUND') {
+      //   attempts.push(`Attempted to load ${pkg}, but it was not installed`)
+      // } else {
+      //   attempts.push(
+      //     `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
+      //   )
+      // }
+    }
   }
+
+  throw attempts
 }
 
 function loadNative() {
+  if (nativeBindings) {
+    return nativeBindings
+  }
+
   let bindings
-  let loadError
+  let attempts = []
 
   for (const triple of triples) {
     try {
       bindings = require(`@next/swc/native/next-swc.${triple.platformArchABI}.node`)
       Log.info('Using locally built binary of @next/swc')
       break
-    } catch (e) {
-      if (e?.code !== 'MODULE_NOT_FOUND') {
-        loadError = e
-      }
-    }
+    } catch (e) {}
   }
 
   if (!bindings) {
     for (const triple of triples) {
+      let pkg = `@next/swc-${triple.platformArchABI}`
       try {
-        bindings = require(`@next/swc-${triple.platformArchABI}`)
+        bindings = require(pkg)
         break
       } catch (e) {
-        if (e?.code !== 'MODULE_NOT_FOUND') {
-          loadError = e
+        if (e?.code === 'MODULE_NOT_FOUND') {
+          attempts.push(`Attempted to load ${pkg}, but it was not installed`)
+        } else {
+          attempts.push(
+            `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
+          )
         }
       }
     }
   }
 
   if (bindings) {
-    return {
+    nativeBindings = {
       isWasm: false,
       transform(src, options) {
         const isModule =
@@ -119,16 +180,10 @@ function loadNative() {
         return bindings.bundle(toBuffer(options))
       },
     }
+    return nativeBindings
   }
 
-  if (loadError) {
-    console.error(loadError)
-  }
-
-  Log.error(
-    `Failed to load SWC binary, see more info here: https://nextjs.org/docs/messages/failed-loading-swc`
-  )
-  process.exit(1)
+  throw attempts
 }
 
 function toBuffer(t) {
@@ -146,7 +201,7 @@ export async function transform(src, options) {
 }
 
 export function transformSync(src, options) {
-  let bindings = loadNative()
+  let bindings = loadBindingsSync()
   return bindings.transformSync(src, options)
 }
 
@@ -156,11 +211,11 @@ export async function minify(src, options) {
 }
 
 export function minifySync(src, options) {
-  let bindings = loadNative()
+  let bindings = loadBindingsSync()
   return bindings.minifySync(src, options)
 }
 
 export async function bundle(options) {
-  let bindings = loadNative()
+  let bindings = loadBindingsSync()
   return bindings.bundle(toBuffer(options))
 }

scripts/setup-wasm.mjs

@@ -0,0 +1,21 @@
+import path from 'path'
+import { readFile, writeFile } from 'fs/promises'
+import { copy } from 'fs-extra'
+;(async function () {
+  let wasmDir = path.join(process.cwd(), 'packages/next-swc/crates/wasm')
+  let wasmTarget = 'nodejs'
+  let wasmPkg = JSON.parse(
+    await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
+  )
+  wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
+
+  await writeFile(
+    path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
+    JSON.stringify(wasmPkg, null, 2)
+  )
+
+  await copy(
+    path.join(wasmDir, `pkg-${wasmTarget}`),
+    path.join(process.cwd(), `node_modules/@next/swc-wasm-${wasmTarget}`)
+  )
+})()

test/integration/production/test/index.test.js

@@ -38,10 +38,16 @@ const context = {}
 describe('Production Usage', () => {
   let output = ''
   beforeAll(async () => {
-    const result = await nextBuild(appDir, undefined, {
+    let opts = {
       stderr: true,
       stdout: true,
-    })
+    }
+    if (process.env.TEST_WASM) {
+      opts.env = {
+        NODE_OPTIONS: '--no-addons',
+      }
+    }
+    const result = await nextBuild(appDir, undefined, opts)
 
     appPort = await findPort()
     context.appPort = appPort