From c628f9253205af60747725faeb2416d501b1169c Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 14 Nov 2022 22:03:17 -0800 Subject: [PATCH 01/14] Update caching for swc turbo builds --- .github/workflows/build_test_deploy.yml | 72 ++++++++++++------------- scripts/normalize-version-bump.js | 66 +++++++++++++++++++++++ 2 files changed, 102 insertions(+), 36 deletions(-) create mode 100755 scripts/normalize-version-bump.js diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index efe7180773c3c01..5dc9ee2f8f96f96 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1030,6 +1030,9 @@ jobs: turbo-${{ github.job }}-${{ github.ref_name }}-${{ steps.get-week.outputs.WEEK }}- turbo-${{ github.job }}-canary-${{ steps.get-week.outputs.WEEK }}- + - name: normalize versions + run: node normalize-version-bump.js + # We use restore-key to pick latest cache. # We will not get exact match, but doc says # "If there are multiple partial matches for a restore key, the action returns the most recently created cache." @@ -1153,17 +1156,17 @@ jobs: target: 'x86_64-apple-darwin' build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi - turbo run build-native --cache-dir=".turbo" -- --release + turbo run build-native -- --release strip -x packages/next-swc/native/next-swc.*.node - host: windows-latest build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" - turbo run build-native --cache-dir=".turbo" -- --release + turbo run build-native -- --release target: 'x86_64-pc-windows-msvc' - host: windows-latest build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" - turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target i686-pc-windows-msvc + turbo run build-native-no-plugin -- --release --target i686-pc-windows-msvc target: 'i686-pc-windows-msvc' - host: ubuntu-latest target: 'x86_64-unknown-linux-gnu' @@ -1175,7 +1178,7 @@ jobs: rustup target add x86_64-unknown-linux-gnu && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi && unset CC_x86_64_unknown_linux_gnu && unset CC && - turbo run build-native --cache-dir=".turbo" -- --release --target x86_64-unknown-linux-gnu && + turbo run build-native -- --release --target x86_64-unknown-linux-gnu && strip packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: 'x86_64-unknown-linux-musl' @@ -1187,7 +1190,7 @@ jobs: rustup default "${RUST_TOOLCHAIN}" && rustup target add x86_64-unknown-linux-musl && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi && - turbo run build-native --cache-dir=".turbo" -- --release --target x86_64-unknown-linux-musl && + turbo run build-native -- --release --target x86_64-unknown-linux-musl && strip packages/next-swc/native/next-swc.*.node - host: macos-latest target: 'aarch64-apple-darwin' @@ -1198,7 +1201,7 @@ jobs: SYSROOT=$(xcrun --sdk macosx --show-sdk-path); export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT"; npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi - turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-apple-darwin + turbo run build-native -- --release --target aarch64-apple-darwin strip -x packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: 'aarch64-unknown-linux-gnu' @@ -1211,7 +1214,7 @@ jobs: rustup target add aarch64-unknown-linux-gnu && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi && export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc && - turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-unknown-linux-gnu && + turbo run build-native -- --release --target aarch64-unknown-linux-gnu && llvm-strip -x packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: 'armv7-unknown-linux-gnueabihf' @@ -1220,7 +1223,7 @@ jobs: sudo apt-get install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf -y build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi - turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target armv7-unknown-linux-gnueabihf + turbo run build-native-no-plugin -- --release --target armv7-unknown-linux-gnueabihf arm-linux-gnueabihf-strip packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: aarch64-linux-android @@ -1235,7 +1238,7 @@ jobs: chmod 777 "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/aarch64/libgcc.a" echo "INPUT(-lunwind)" > "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/aarch64/libgcc.a" npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi - turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-linux-android + turbo run build-native -- --release --target aarch64-linux-android ${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: armv7-linux-androideabi @@ -1250,7 +1253,7 @@ jobs: chmod 777 "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/arm/libgcc.a" echo "INPUT(-lunwind)" > "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/arm/libgcc.a" npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" - turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target armv7-linux-androideabi + turbo run build-native-no-plugin -- --release --target armv7-linux-androideabi ${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: 'aarch64-unknown-linux-musl' @@ -1263,17 +1266,21 @@ jobs: rustup toolchain install "${RUST_TOOLCHAIN}" && rustup default "${RUST_TOOLCHAIN}" && rustup target add aarch64-unknown-linux-musl && - turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-unknown-linux-musl && + turbo run build-native -- --release --target aarch64-unknown-linux-musl && llvm-strip -x packages/next-swc/native/next-swc.*.node - host: windows-latest target: 'aarch64-pc-windows-msvc' build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" - turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target aarch64-pc-windows-msvc --cargo-flags=--no-default-features + turbo run build-native-no-plugin -- --release --target aarch64-pc-windows-msvc --cargo-flags=--no-default-features if: ${{ needs.build.outputs.isRelease == 'true' }} needs: build name: stable - ${{ matrix.settings.target }} - node@16 runs-on: ${{ matrix.settings.host }} + env: + TURBO_TEAM: 'vercel' + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_REMOTE_ONLY: 'true' steps: # https://github.com/actions/virtual-environments/issues/1187 - name: tune linux network @@ -1302,16 +1309,6 @@ jobs: key: next-swc-cargo-cache-${{ matrix.settings.target }}--${{ hashFiles('**/Cargo.lock') }} restore-keys: | next-swc-cargo-cache-${{ matrix.settings.target }} - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v3 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ matrix.settings.target }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ github.ref_name }}-${{ matrix.settings.target }} - turbo-${{ github.job }}-${{ github.ref_name }}-${{ matrix.settings.target }}-${{ needs.build.outputs.weekNum }}- - turbo-${{ github.job }}-canary-${{ matrix.settings.target }}-${{ needs.build.outputs.weekNum }}- - name: Setup node uses: actions/setup-node@v3 @@ -1341,6 +1338,9 @@ jobs: path: ~/.cargo/git key: ${{ matrix.settings.target }}-cargo-index + - name: normalize versions + run: node normalize-version-bump.js + - name: Setup toolchain run: ${{ matrix.settings.setup }} if: ${{ matrix.settings.setup }} @@ -1370,6 +1370,10 @@ jobs: needs: build name: stable - x86_64-unknown-freebsd - node@16 runs-on: macos-12 + env: + TURBO_TEAM: 'vercel' + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_REMOTE_ONLY: 'true' steps: - name: tune mac network run: sudo sysctl -w net.link.generic.system.hwcksum_tx=0 && sudo sysctl -w net.link.generic.system.hwcksum_rx=0 @@ -1393,7 +1397,7 @@ jobs: # Disable LTO, or the lld may crash with OOM CARGO_PROFILE_RELEASE_LTO: false with: - envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION TURBO_VERSION RUST_TOOLCHAIN PNPM_VERSION + envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION TURBO_VERSION RUST_TOOLCHAIN PNPM_VERSION TURBO_TEAM TURBO_REMOTE_ONLY TURBO_TOKEN usesh: true mem: 6000 prepare: | @@ -1414,6 +1418,7 @@ jobs: whoami env freebsd-version + node normalize-version-bump.js pnpm --filter=@next/swc run build-native-no-plugin --platform --release --target x86_64-unknown-freebsd rm -rf node_modules rm -rf packages/next-swc/target @@ -1431,6 +1436,10 @@ jobs: matrix: target: [web, nodejs] runs-on: macos-latest + env: + TURBO_TEAM: 'vercel' + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_REMOTE_ONLY: 'true' steps: - uses: actions/cache@v3 id: restore-build @@ -1454,22 +1463,14 @@ jobs: - run: npm i -g turbo@${{ env.TURBO_VERSION }} pnpm@${PNPM_VERSION} - - name: Turbo cache - id: turbo-cache - uses: actions/cache@v3 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ matrix.target }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ matrix.target }}- - turbo-${{ github.job }}-${{ matrix.target }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}- - turbo-${{ github.job }}-${{ matrix.target }}-canary-${{ needs.build.outputs.weekNum }}- - - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: normalize versions + run: node normalize-version-bump.js + - name: Build - run: turbo run build-wasm --cache-dir=".turbo" -- --target ${{ matrix.target }} + run: turbo run build-wasm -- --target ${{ matrix.target }} - name: Add target to folder name run: '[[ -d "packages/next-swc/crates/wasm/pkg" ]] && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-${{ matrix.target }} || ls packages/next-swc/crates/wasm' @@ -1513,7 +1514,6 @@ jobs: id: turbo-cache uses: actions/cache@v3 if: ${{needs.build.outputs.docsChange == 'nope'}} - with: path: .turbo key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ steps.get-week.outputs.WEEK }}-${{ github.sha }} diff --git a/scripts/normalize-version-bump.js b/scripts/normalize-version-bump.js new file mode 100755 index 000000000000000..df455398a3710f3 --- /dev/null +++ b/scripts/normalize-version-bump.js @@ -0,0 +1,66 @@ +#!/usr/bin/env node +// @ts-check + +/* + This prevents busting the turbo cache un-necessarily due + to bumping the version in the repo's package.json files +*/ +const path = require('path') +const fs = require('fs/promises') + +const cwd = process.cwd() +const NORMALIZED_VERSION = '0.0.0' + +const readJson = async (filePath) => + JSON.parse(await fs.readFile(filePath, 'utf8')) + +const writeJson = async (filePath, data) => + fs.writeFile(filePath, JSON.stringify(data, null, 2) + '\n') + +;(async function () { + const packages = await fs.readdir(path.join(cwd, 'packages')) + const pkgJsonData = new Map() + const pkgNames = [] + await Promise.all( + packages.map(async (pkgDir) => { + const data = await readJson( + path.join(cwd, 'packages', pkgDir, 'package.json') + ) + pkgNames.push(data.name) + pkgJsonData.set(pkgDir, data) + }) + ) + const normalizeVersions = async (filePath, data) => { + data = data || (await readJson(filePath)) + const version = data.version + + if (version) { + data.version = NORMALIZED_VERSION + const normalizeEntry = (type, key) => { + const pkgVersion = data[type][key] + + if (pkgNames.includes(key) && pkgVersion === version) { + data[type][key] = NORMALIZED_VERSION + } + } + for (const key of Object.keys(data.dependencies || {})) { + normalizeEntry('dependencies', key) + } + for (const key of Object.keys(data.devDependencies || {})) { + normalizeEntry('devDependencies', key) + } + await writeJson(filePath, data) + } + } + await Promise.all( + packages.map((pkgDir) => + normalizeVersions( + path.join(cwd, 'packages', pkgDir, 'package.json'), + pkgJsonData.get(pkgDir) + ) + ) + ) + await normalizeVersions(path.join(cwd, 'lerna.json')) + await fs.unlink(path.join(cwd, 'pnpm-lock.yaml')) + await fs.writeFile(path.join(cwd, 'pnpm-lock.yaml'), '') +})() From 15c9bcae41c301b67507233cc6a5512669fc3c24 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 14 Nov 2022 22:04:54 -0800 Subject: [PATCH 02/14] test builds --- .github/workflows/build_test_deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 5dc9ee2f8f96f96..3aa836d63686301 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1273,7 +1273,7 @@ jobs: build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" turbo run build-native-no-plugin -- --release --target aarch64-pc-windows-msvc --cargo-flags=--no-default-features - if: ${{ needs.build.outputs.isRelease == 'true' }} + # if: ${{ needs.build.outputs.isRelease == 'true' }} needs: build name: stable - ${{ matrix.settings.target }} - node@16 runs-on: ${{ matrix.settings.host }} @@ -1366,7 +1366,7 @@ jobs: path: packages/next-swc/native/next-swc.*.node build-native-freebsd: - if: ${{ needs.build.outputs.isRelease == 'true' }} + # if: ${{ needs.build.outputs.isRelease == 'true' }} needs: build name: stable - x86_64-unknown-freebsd - node@16 runs-on: macos-12 @@ -1431,7 +1431,7 @@ jobs: build-wasm: needs: build - if: ${{ needs.build.outputs.isRelease == 'true' }} + # if: ${{ needs.build.outputs.isRelease == 'true' }} strategy: matrix: target: [web, nodejs] From a8e8677b785a342da8565a45f6297b32b75aa63b Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 14 Nov 2022 22:08:15 -0800 Subject: [PATCH 03/14] update path --- .github/workflows/build_test_deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 3aa836d63686301..6ccfc501dc18320 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1031,7 +1031,7 @@ jobs: turbo-${{ github.job }}-canary-${{ steps.get-week.outputs.WEEK }}- - name: normalize versions - run: node normalize-version-bump.js + run: node scripts/normalize-version-bump.js # We use restore-key to pick latest cache. # We will not get exact match, but doc says @@ -1339,7 +1339,7 @@ jobs: key: ${{ matrix.settings.target }}-cargo-index - name: normalize versions - run: node normalize-version-bump.js + run: node scripts/normalize-version-bump.js - name: Setup toolchain run: ${{ matrix.settings.setup }} @@ -1418,7 +1418,7 @@ jobs: whoami env freebsd-version - node normalize-version-bump.js + node scripts/normalize-version-bump.js pnpm --filter=@next/swc run build-native-no-plugin --platform --release --target x86_64-unknown-freebsd rm -rf node_modules rm -rf packages/next-swc/target @@ -1467,7 +1467,7 @@ jobs: run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: normalize versions - run: node normalize-version-bump.js + run: node scripts/normalize-version-bump.js - name: Build run: turbo run build-wasm -- --target ${{ matrix.target }} From 2ed2ca65ae480e41ee6fa56ae6e3c4f472221ab7 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 14 Nov 2022 22:19:29 -0800 Subject: [PATCH 04/14] keep scripts for freebsd --- .github/workflows/build_test_deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 6ccfc501dc18320..2799dbcec0f2a67 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1384,7 +1384,6 @@ jobs: rm -rf docs rm -rf errors rm -rf examples - rm -rf scripts rm -rf test - name: Build id: build From 15dfd3fb8472c60867e509ecfec6786b42273636 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 14 Nov 2022 22:20:55 -0800 Subject: [PATCH 05/14] normalize version on wasm-dev build --- .github/workflows/build_test_deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 2799dbcec0f2a67..009035e3940da84 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1525,6 +1525,9 @@ jobs: if: ${{needs.build.outputs.docsChange == 'nope'}} run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: normalize versions + run: node scripts/normalize-version-bump.js + - name: Build if: ${{needs.build.outputs.docsChange == 'nope'}} run: turbo run build-wasm --cache-dir=".turbo" -- --target nodejs --dev From 1ea54326b0cef6045e8716a89573cae735d3151a Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 14 Nov 2022 22:40:13 -0800 Subject: [PATCH 06/14] update version --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 009035e3940da84..0b67e2edd920709 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1387,7 +1387,7 @@ jobs: rm -rf test - name: Build id: build - uses: vmactions/freebsd-vm@v0.3.0 + uses: vmactions/freebsd-vm@v0 env: DEBUG: napi:* RUSTUP_HOME: /usr/local/rustup From 76bd0636896905f50247cab076a20a749fb79630 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 14 Nov 2022 23:52:02 -0800 Subject: [PATCH 07/14] bump From eea77d6611aa303a230b3068bc0708df474b576c Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 15 Nov 2022 00:08:49 -0800 Subject: [PATCH 08/14] update env --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 0b67e2edd920709..8c0468df0d670f5 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1351,7 +1351,7 @@ jobs: if: ${{ matrix.settings.docker }} with: image: ${{ matrix.settings.docker }} - options: -e RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -e NAPI_CLI_VERSION=${{ env.NAPI_CLI_VERSION }} -e TURBO_VERSION=${{ env.TURBO_VERSION }} -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build + options: -e RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -e NAPI_CLI_VERSION=${{ env.NAPI_CLI_VERSION }} -e TURBO_VERSION=${{ env.TURBO_VERSION }} -e TURBO_TEAM=vercel -e TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} -e TURBO_REMOTE_ONLY=true -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build run: ${{ matrix.settings.build }} - name: 'Build' From faae442f033591d5e337c4bd6d6d6927566d457e Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 15 Nov 2022 00:12:21 -0800 Subject: [PATCH 09/14] update freebsd --- .github/workflows/build_test_deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 8c0468df0d670f5..eab921e494134b4 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1402,7 +1402,7 @@ jobs: prepare: | pkg install -y curl node16 curl -qL https://www.npmjs.com/install.sh | sh - npm i -g pnpm@${PNPM_VERSION} "@napi-rs/cli@${NAPI_CLI_VERSION}" + npm i -g pnpm@${PNPM_VERSION} "@napi-rs/cli@${NAPI_CLI_VERSION}" turbo@${TURBO_VERSION} curl https://sh.rustup.rs -sSf --output rustup.sh sh rustup.sh -y --profile minimal --default-toolchain stable export PATH="/usr/local/cargo/bin:$PATH" @@ -1418,7 +1418,7 @@ jobs: env freebsd-version node scripts/normalize-version-bump.js - pnpm --filter=@next/swc run build-native-no-plugin --platform --release --target x86_64-unknown-freebsd + turbo run build-native-no-plugin -- --platform --release --target x86_64-unknown-freebsd rm -rf node_modules rm -rf packages/next-swc/target - name: Upload artifact From 3d7c21f4a5187d167b4d6b50d55f50b144d2d308 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 15 Nov 2022 00:59:56 -0800 Subject: [PATCH 10/14] add workaround for freebsd caching --- .github/workflows/build_test_deploy.yml | 32 ++++++++++++++++++++++--- packages/next-swc/package.json | 3 ++- scripts/pull-freebsd-cache.js | 30 +++++++++++++++++++++++ turbo.json | 4 ++++ 4 files changed, 65 insertions(+), 4 deletions(-) create mode 100755 scripts/pull-freebsd-cache.js diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index eab921e494134b4..5b0bda811ebcf1c 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1378,6 +1378,26 @@ jobs: - name: tune mac network run: sudo sysctl -w net.link.generic.system.hwcksum_tx=0 && sudo sysctl -w net.link.generic.system.hwcksum_rx=0 - uses: actions/checkout@v3 + + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version: 16 + check-latest: true + + - run: npm i -g turbo@${{ env.TURBO_VERSION }} pnpm@${PNPM_VERSION} + + # attempt pulling existing turbo cache, this must be done + # outside of the freeBSD vm as there isn't a turbo build + # for freeBSD currently + - run: node ./script/pull-freebsd-cache.js + + - name: check build exists + run: if [ -f packages/next-swc/native/next-swc.freebsd-x64.node ]; then echo "::set-output name=BUILD_EXISTS::yes"; else echo "::set-output name=BUILD_EXISTS::no"; fi + id: build-exists + + - run: echo ${{steps.build-exists.outputs.BUILD_EXISTS}} + - name: Delete useless files run: | rm -rf bench @@ -1388,6 +1408,7 @@ jobs: - name: Build id: build uses: vmactions/freebsd-vm@v0 + if: ${{ steps.build-exists.outputs.BUILD_EXISTS == 'no' }} env: DEBUG: napi:* RUSTUP_HOME: /usr/local/rustup @@ -1396,13 +1417,13 @@ jobs: # Disable LTO, or the lld may crash with OOM CARGO_PROFILE_RELEASE_LTO: false with: - envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION TURBO_VERSION RUST_TOOLCHAIN PNPM_VERSION TURBO_TEAM TURBO_REMOTE_ONLY TURBO_TOKEN + envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION RUST_TOOLCHAIN PNPM_VERSION usesh: true mem: 6000 prepare: | pkg install -y curl node16 curl -qL https://www.npmjs.com/install.sh | sh - npm i -g pnpm@${PNPM_VERSION} "@napi-rs/cli@${NAPI_CLI_VERSION}" turbo@${TURBO_VERSION} + npm i -g pnpm@${PNPM_VERSION} "@napi-rs/cli@${NAPI_CLI_VERSION}" curl https://sh.rustup.rs -sSf --output rustup.sh sh rustup.sh -y --profile minimal --default-toolchain stable export PATH="/usr/local/cargo/bin:$PATH" @@ -1418,9 +1439,14 @@ jobs: env freebsd-version node scripts/normalize-version-bump.js - turbo run build-native-no-plugin -- --platform --release --target x86_64-unknown-freebsd + pnpm --filter=@next/swc run build-native-no-plugin --platform --release --target x86_64-unknown-freebsd rm -rf node_modules rm -rf packages/next-swc/target + + - name: cache build + if: ${{ steps.build-exists.outputs.BUILD_EXISTS == 'no' }} + run: turbo run cache-build-native -- --platform --release --target x86_64-unknown-freebsd + - name: Upload artifact uses: actions/upload-artifact@v3 with: diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index d6aa012d947ebd8..eb097c192c5ab3b 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -5,7 +5,8 @@ "scripts": { "build-native": "napi build --platform -p next-swc-napi --cargo-name next_swc_napi --features plugin --js false native", "build-native-no-plugin": "napi build --platform -p next-swc-napi --cargo-name next_swc_napi --js false native", - "build-wasm": "wasm-pack build crates/wasm --scope=next" + "build-wasm": "wasm-pack build crates/wasm --scope=next", + "cache-build-native": "echo $(ls native)" }, "napi": { "name": "next-swc", diff --git a/scripts/pull-freebsd-cache.js b/scripts/pull-freebsd-cache.js new file mode 100755 index 000000000000000..8e26caaca230cb2 --- /dev/null +++ b/scripts/pull-freebsd-cache.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node +// @ts-check + +const { execSync } = require('child_process') + +;(async function () { + const turboResult = execSync( + `pnpm turbo run cache-build-native --dry=json -- --platform --release --target x86_64-unknown-freebsd` + ).toString() + + const turboData = JSON.parse(turboResult) + + const task = turboData.tasks.find((t) => t.command !== '') + + if (!task) { + console.warn(`Failed to find related turbo task`, turboResult) + return + } + + // pull cache if it was available + if (task.cacheState.local || task.cacheState.remote) { + const pullResult = execSync( + `pnpm turbo run cache-build-native -- --platform --release --target x86_64-unknown-freebsd` + ).toString() + console.log(pullResult) + } else { + console.warn(`No turbo cache was available, continuing...`) + console.warn(task) + } +})() diff --git a/turbo.json b/turbo.json index 2b9677cb6bea5c4..8b9d0365efa817d 100644 --- a/turbo.json +++ b/turbo.json @@ -13,6 +13,10 @@ "dependsOn": ["^build-wasm"], "outputs": ["crates/wasm/pkg/*"] }, + "cache-build-native": { + "dependsOn": ["^cache-build-native"], + "outputs": ["native/*.node"] + }, "build": { "dependsOn": ["^build"], "outputs": ["dist/**"] From 16a08c6418a4e09ccd5f4523f43df88f81db28f9 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 15 Nov 2022 01:23:13 -0800 Subject: [PATCH 11/14] update path --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 5b0bda811ebcf1c..fffe32acc00c50a 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1390,7 +1390,7 @@ jobs: # attempt pulling existing turbo cache, this must be done # outside of the freeBSD vm as there isn't a turbo build # for freeBSD currently - - run: node ./script/pull-freebsd-cache.js + - run: node ./scripts/pull-freebsd-cache.js - name: check build exists run: if [ -f packages/next-swc/native/next-swc.freebsd-x64.node ]; then echo "::set-output name=BUILD_EXISTS::yes"; else echo "::set-output name=BUILD_EXISTS::no"; fi From 86c094b8ff35f68f3be974795f4a1d1244552421 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 15 Nov 2022 02:35:18 -0800 Subject: [PATCH 12/14] bump From ffe8c493d7daf3860a29f33b046328bb4dbc7173 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 15 Nov 2022 03:14:45 -0800 Subject: [PATCH 13/14] fix cache miss --- .github/workflows/build_test_deploy.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index fffe32acc00c50a..dce78ade668d03f 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1387,6 +1387,16 @@ jobs: - run: npm i -g turbo@${{ env.TURBO_VERSION }} pnpm@${PNPM_VERSION} + - name: Delete useless files + run: | + rm -rf bench + rm -rf docs + rm -rf errors + rm -rf examples + rm -rf test + + - run: node scripts/normalize-version-bump.js + # attempt pulling existing turbo cache, this must be done # outside of the freeBSD vm as there isn't a turbo build # for freeBSD currently @@ -1398,13 +1408,6 @@ jobs: - run: echo ${{steps.build-exists.outputs.BUILD_EXISTS}} - - name: Delete useless files - run: | - rm -rf bench - rm -rf docs - rm -rf errors - rm -rf examples - rm -rf test - name: Build id: build uses: vmactions/freebsd-vm@v0 @@ -1438,14 +1441,13 @@ jobs: whoami env freebsd-version - node scripts/normalize-version-bump.js pnpm --filter=@next/swc run build-native-no-plugin --platform --release --target x86_64-unknown-freebsd rm -rf node_modules rm -rf packages/next-swc/target - name: cache build if: ${{ steps.build-exists.outputs.BUILD_EXISTS == 'no' }} - run: turbo run cache-build-native -- --platform --release --target x86_64-unknown-freebsd + run: pnpm turbo run cache-build-native -- --platform --release --target x86_64-unknown-freebsd - name: Upload artifact uses: actions/upload-artifact@v3 From 0b4129aee38f283a29aa808509ac4b332d7e1ee8 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 15 Nov 2022 03:23:57 -0800 Subject: [PATCH 14/14] re-add isRelease checks --- .github/workflows/build_test_deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index dce78ade668d03f..945afa245cf5165 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1273,7 +1273,7 @@ jobs: build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" turbo run build-native-no-plugin -- --release --target aarch64-pc-windows-msvc --cargo-flags=--no-default-features - # if: ${{ needs.build.outputs.isRelease == 'true' }} + if: ${{ needs.build.outputs.isRelease == 'true' }} needs: build name: stable - ${{ matrix.settings.target }} - node@16 runs-on: ${{ matrix.settings.host }} @@ -1366,7 +1366,7 @@ jobs: path: packages/next-swc/native/next-swc.*.node build-native-freebsd: - # if: ${{ needs.build.outputs.isRelease == 'true' }} + if: ${{ needs.build.outputs.isRelease == 'true' }} needs: build name: stable - x86_64-unknown-freebsd - node@16 runs-on: macos-12 @@ -1458,7 +1458,7 @@ jobs: build-wasm: needs: build - # if: ${{ needs.build.outputs.isRelease == 'true' }} + if: ${{ needs.build.outputs.isRelease == 'true' }} strategy: matrix: target: [web, nodejs]