From 21f8c6a7c3bc9eb5146b7e19cee85c663b555314 Mon Sep 17 00:00:00 2001 From: Maia Teegarden Date: Tue, 7 Dec 2021 02:14:38 -0800 Subject: [PATCH] Chore/load bindings improvements (#32191) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- .github/workflows/build_test_deploy.yml | 106 ++++++++++++++++- packages/next/build/swc/index.js | 111 +++++++++++++----- scripts/setup-wasm.mjs | 21 ++++ .../integration/production/test/index.test.js | 10 +- 4 files changed, 217 insertions(+), 31 deletions(-) create mode 100644 scripts/setup-wasm.mjs diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 73d088af979827c..35f425d4f2d01e4 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -715,6 +715,50 @@ jobs: - run: cd packages/next-swc && cargo test if: ${{ steps.docs-change.outputs.DOCS_CHANGE != 'docs only change' }} + test-wasm: + name: Test the wasm build + runs-on: ubuntu-18.04 + needs: [build, build-native-dev, build-wasm-dev] + + steps: + - uses: actions/cache@v2 + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + id: restore-build + with: + path: ./* + key: ${{ github.sha }} + + - uses: actions/download-artifact@v2 + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + with: + name: wasm-dev-binary + path: packages/next-swc/crates/wasm/pkg-nodejs + + - run: ls packages/next-swc/crates/wasm + + - uses: actions/download-artifact@v2 + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + with: + name: next-swc-dev-binary + path: packages/next-swc/native + + # node version needs to be 16+ to use --no-addons option + - name: Setup node + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + uses: actions/setup-node@v2 + with: + node-version: 16 + check-latest: true + + - run: npm i -g playwright-chromium@1.14.1 && npx playwright install-deps + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + + - run: node ./scripts/setup-wasm.mjs + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + + - run: TEST_WASM=true xvfb-run node run-tests.js test/integration/production/test/index.test.js + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + # Build binaries for publishing build-native: needs: build @@ -1255,4 +1299,64 @@ jobs: name: wasm-binaries path: packages/next-swc/crates/wasm/pkg-* - - run: ls packages/next-swc/crates/wasm + build-wasm-dev: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v2 + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + id: restore-build + with: + path: ./* + key: ${{ github.sha }}-${{ github.run_number }}-${{ github.run_attempt }} + + - name: Setup node + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + uses: actions/setup-node@v2 + with: + node-version: 14 + + - name: Install Rust + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly-2021-11-15 + override: true + target: wasm32-unknown-unknown + + - name: Cache + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + uses: actions/cache@v2 + with: + path: | + ~/.cargo/ + **/target/ + key: ${{ runner.os }}-publish-integration + + - name: Cache wasm binary + if: ${{needs.build.outputs.docsChange != 'docs only change'}} + id: binary-cache + uses: actions/cache@v2 + with: + path: packages/next-swc/crates/wasm/pkg-nodejs + key: dev-wasm-next-swc-nightly-2021-11-15-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }} + + - name: Install wasm-pack + if: ${{needs.build.outputs.docsChange != 'docs only change' && steps.binary-cache.outputs.cache-hit != 'true'}} + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + + - name: Build + if: ${{needs.build.outputs.docsChange != 'docs only change' && steps.binary-cache.outputs.cache-hit != 'true'}} + run: (wasm-pack build packages/next-swc/crates/wasm --dev --scope=next --target nodejs) + + - name: Add target to folder name + if: ${{needs.build.outputs.docsChange != 'docs only change' && steps.binary-cache.outputs.cache-hit != 'true'}} + run: mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-nodejs + + - name: Upload artifact + if: ${{needs.build.outputs.docsChange != 'docs only change' && steps.binary-cache.outputs.cache-hit != 'true'}} + uses: actions/upload-artifact@v2 + with: + name: wasm-dev-binary + path: packages/next-swc/crates/wasm/pkg-nodejs diff --git a/packages/next/build/swc/index.js b/packages/next/build/swc/index.js index 85b2e35d3181e63..74014aee82ffd4a 100644 --- a/packages/next/build/swc/index.js +++ b/packages/next/build/swc/index.js @@ -6,19 +6,63 @@ const ArchName = arch() const PlatformName = platform() const triples = platformArchTriples[PlatformName][ArchName] || [] +let nativeBindings +let wasmBindings + async function loadBindings() { - return (await loadWasm()) || loadNative() + let attempts = [] + try { + return loadNative() + } catch (a) { + attempts = attempts.concat(a) + } + + try { + let bindings = await loadWasm() + return bindings + } catch (a) { + attempts = attempts.concat(a) + } + + logLoadFailure(attempts) +} + +function loadBindingsSync() { + let attempts = [] + try { + return loadNative() + } catch (a) { + attempts = attempts.concat(a) + } + + logLoadFailure(attempts) +} + +function logLoadFailure(attempts) { + for (let attempt of attempts) { + Log.info(attempt) + } + + Log.error( + `Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc` + ) + process.exit(1) } async function loadWasm() { - // Try to load wasm bindings - for (let specifier of ['@next/swc-wasm-web', '@next/swc-wasm-nodejs']) { + if (wasmBindings) { + return wasmBindings + } + + let attempts = [] + for (let pkg of ['@next/swc-wasm-nodejs', '@next/swc-wasm-web']) { try { - let bindings = await import(specifier) - if (specifier === '@next/swc-wasm-web') { + let bindings = await import(pkg) + if (pkg === '@next/swc-wasm-web') { bindings = await bindings.default() } - return { + Log.info('Using experimental wasm build of next-swc') + wasmBindings = { isWasm: true, transform(src, options) { return Promise.resolve( @@ -29,41 +73,58 @@ async function loadWasm() { return Promise.resolve(bindings.minifySync(src.toString(), options)) }, } - } catch (e) {} + return wasmBindings + } catch (e) { + // Do not report attempts to load wasm when it is still experimental + // if (e?.code === 'ERR_MODULE_NOT_FOUND') { + // attempts.push(`Attempted to load ${pkg}, but it was not installed`) + // } else { + // attempts.push( + // `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}` + // ) + // } + } } + + throw attempts } function loadNative() { + if (nativeBindings) { + return nativeBindings + } + let bindings - let loadError + let attempts = [] for (const triple of triples) { try { bindings = require(`@next/swc/native/next-swc.${triple.platformArchABI}.node`) Log.info('Using locally built binary of @next/swc') break - } catch (e) { - if (e?.code !== 'MODULE_NOT_FOUND') { - loadError = e - } - } + } catch (e) {} } if (!bindings) { for (const triple of triples) { + let pkg = `@next/swc-${triple.platformArchABI}` try { - bindings = require(`@next/swc-${triple.platformArchABI}`) + bindings = require(pkg) break } catch (e) { - if (e?.code !== 'MODULE_NOT_FOUND') { - loadError = e + if (e?.code === 'MODULE_NOT_FOUND') { + attempts.push(`Attempted to load ${pkg}, but it was not installed`) + } else { + attempts.push( + `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}` + ) } } } } if (bindings) { - return { + nativeBindings = { isWasm: false, transform(src, options) { const isModule = @@ -119,16 +180,10 @@ function loadNative() { return bindings.bundle(toBuffer(options)) }, } + return nativeBindings } - if (loadError) { - console.error(loadError) - } - - Log.error( - `Failed to load SWC binary, see more info here: https://nextjs.org/docs/messages/failed-loading-swc` - ) - process.exit(1) + throw attempts } function toBuffer(t) { @@ -146,7 +201,7 @@ export async function transform(src, options) { } export function transformSync(src, options) { - let bindings = loadNative() + let bindings = loadBindingsSync() return bindings.transformSync(src, options) } @@ -156,11 +211,11 @@ export async function minify(src, options) { } export function minifySync(src, options) { - let bindings = loadNative() + let bindings = loadBindingsSync() return bindings.minifySync(src, options) } export async function bundle(options) { - let bindings = loadNative() + let bindings = loadBindingsSync() return bindings.bundle(toBuffer(options)) } diff --git a/scripts/setup-wasm.mjs b/scripts/setup-wasm.mjs new file mode 100644 index 000000000000000..720cd8930163860 --- /dev/null +++ b/scripts/setup-wasm.mjs @@ -0,0 +1,21 @@ +import path from 'path' +import { readFile, writeFile } from 'fs/promises' +import { copy } from 'fs-extra' +;(async function () { + let wasmDir = path.join(process.cwd(), 'packages/next-swc/crates/wasm') + let wasmTarget = 'nodejs' + let wasmPkg = JSON.parse( + await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`)) + ) + wasmPkg.name = `@next/swc-wasm-${wasmTarget}` + + await writeFile( + path.join(wasmDir, `pkg-${wasmTarget}/package.json`), + JSON.stringify(wasmPkg, null, 2) + ) + + await copy( + path.join(wasmDir, `pkg-${wasmTarget}`), + path.join(process.cwd(), `node_modules/@next/swc-wasm-${wasmTarget}`) + ) +})() diff --git a/test/integration/production/test/index.test.js b/test/integration/production/test/index.test.js index 4c887a89ab97558..04a28433f9f037e 100644 --- a/test/integration/production/test/index.test.js +++ b/test/integration/production/test/index.test.js @@ -38,10 +38,16 @@ const context = {} describe('Production Usage', () => { let output = '' beforeAll(async () => { - const result = await nextBuild(appDir, undefined, { + let opts = { stderr: true, stdout: true, - }) + } + if (process.env.TEST_WASM) { + opts.env = { + NODE_OPTIONS: '--no-addons', + } + } + const result = await nextBuild(appDir, undefined, opts) appPort = await findPort() context.appPort = appPort