From ea9c5aac5f23ebe526d41300e2e3800fc4f4a7a8 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 15 Nov 2022 08:24:04 -0800 Subject: [PATCH] Update caching for swc turbo builds (#42929) Updates to leverage turbo remote caching for the swc builds and normalize the package versions to prevent repo version bumps from un-necessarily invalidating the cache. Full cache hits can be seen in this test run https://github.com/vercel/next.js/actions/runs/3469932131/jobs/5797559609 --- .github/workflows/build_test_deploy.yml | 108 +++++++++++++++--------- packages/next-swc/package.json | 3 +- scripts/normalize-version-bump.js | 66 +++++++++++++++ scripts/pull-freebsd-cache.js | 30 +++++++ turbo.json | 4 + 5 files changed, 171 insertions(+), 40 deletions(-) create mode 100755 scripts/normalize-version-bump.js create mode 100755 scripts/pull-freebsd-cache.js diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index efe7180773c3c01..945afa245cf5165 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1030,6 +1030,9 @@ jobs: turbo-${{ github.job }}-${{ github.ref_name }}-${{ steps.get-week.outputs.WEEK }}- turbo-${{ github.job }}-canary-${{ steps.get-week.outputs.WEEK }}- + - name: normalize versions + run: node scripts/normalize-version-bump.js + # We use restore-key to pick latest cache. # We will not get exact match, but doc says # "If there are multiple partial matches for a restore key, the action returns the most recently created cache." @@ -1153,17 +1156,17 @@ jobs: target: 'x86_64-apple-darwin' build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi - turbo run build-native --cache-dir=".turbo" -- --release + turbo run build-native -- --release strip -x packages/next-swc/native/next-swc.*.node - host: windows-latest build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" - turbo run build-native --cache-dir=".turbo" -- --release + turbo run build-native -- --release target: 'x86_64-pc-windows-msvc' - host: windows-latest build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" - turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target i686-pc-windows-msvc + turbo run build-native-no-plugin -- --release --target i686-pc-windows-msvc target: 'i686-pc-windows-msvc' - host: ubuntu-latest target: 'x86_64-unknown-linux-gnu' @@ -1175,7 +1178,7 @@ jobs: rustup target add x86_64-unknown-linux-gnu && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi && unset CC_x86_64_unknown_linux_gnu && unset CC && - turbo run build-native --cache-dir=".turbo" -- --release --target x86_64-unknown-linux-gnu && + turbo run build-native -- --release --target x86_64-unknown-linux-gnu && strip packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: 'x86_64-unknown-linux-musl' @@ -1187,7 +1190,7 @@ jobs: rustup default "${RUST_TOOLCHAIN}" && rustup target add x86_64-unknown-linux-musl && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi && - turbo run build-native --cache-dir=".turbo" -- --release --target x86_64-unknown-linux-musl && + turbo run build-native -- --release --target x86_64-unknown-linux-musl && strip packages/next-swc/native/next-swc.*.node - host: macos-latest target: 'aarch64-apple-darwin' @@ -1198,7 +1201,7 @@ jobs: SYSROOT=$(xcrun --sdk macosx --show-sdk-path); export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT"; npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi - turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-apple-darwin + turbo run build-native -- --release --target aarch64-apple-darwin strip -x packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: 'aarch64-unknown-linux-gnu' @@ -1211,7 +1214,7 @@ jobs: rustup target add aarch64-unknown-linux-gnu && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi && export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc && - turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-unknown-linux-gnu && + turbo run build-native -- --release --target aarch64-unknown-linux-gnu && llvm-strip -x packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: 'armv7-unknown-linux-gnueabihf' @@ -1220,7 +1223,7 @@ jobs: sudo apt-get install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf -y build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi - turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target armv7-unknown-linux-gnueabihf + turbo run build-native-no-plugin -- --release --target armv7-unknown-linux-gnueabihf arm-linux-gnueabihf-strip packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: aarch64-linux-android @@ -1235,7 +1238,7 @@ jobs: chmod 777 "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/aarch64/libgcc.a" echo "INPUT(-lunwind)" > "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/aarch64/libgcc.a" npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi - turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-linux-android + turbo run build-native -- --release --target aarch64-linux-android ${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: armv7-linux-androideabi @@ -1250,7 +1253,7 @@ jobs: chmod 777 "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/arm/libgcc.a" echo "INPUT(-lunwind)" > "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/arm/libgcc.a" npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" - turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target armv7-linux-androideabi + turbo run build-native-no-plugin -- --release --target armv7-linux-androideabi ${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip packages/next-swc/native/next-swc.*.node - host: ubuntu-latest target: 'aarch64-unknown-linux-musl' @@ -1263,17 +1266,21 @@ jobs: rustup toolchain install "${RUST_TOOLCHAIN}" && rustup default "${RUST_TOOLCHAIN}" && rustup target add aarch64-unknown-linux-musl && - turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-unknown-linux-musl && + turbo run build-native -- --release --target aarch64-unknown-linux-musl && llvm-strip -x packages/next-swc/native/next-swc.*.node - host: windows-latest target: 'aarch64-pc-windows-msvc' build: | npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}" - turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target aarch64-pc-windows-msvc --cargo-flags=--no-default-features + turbo run build-native-no-plugin -- --release --target aarch64-pc-windows-msvc --cargo-flags=--no-default-features if: ${{ needs.build.outputs.isRelease == 'true' }} needs: build name: stable - ${{ matrix.settings.target }} - node@16 runs-on: ${{ matrix.settings.host }} + env: + TURBO_TEAM: 'vercel' + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_REMOTE_ONLY: 'true' steps: # https://github.com/actions/virtual-environments/issues/1187 - name: tune linux network @@ -1302,16 +1309,6 @@ jobs: key: next-swc-cargo-cache-${{ matrix.settings.target }}--${{ hashFiles('**/Cargo.lock') }} restore-keys: | next-swc-cargo-cache-${{ matrix.settings.target }} - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v3 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ matrix.settings.target }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ github.ref_name }}-${{ matrix.settings.target }} - turbo-${{ github.job }}-${{ github.ref_name }}-${{ matrix.settings.target }}-${{ needs.build.outputs.weekNum }}- - turbo-${{ github.job }}-canary-${{ matrix.settings.target }}-${{ needs.build.outputs.weekNum }}- - name: Setup node uses: actions/setup-node@v3 @@ -1341,6 +1338,9 @@ jobs: path: ~/.cargo/git key: ${{ matrix.settings.target }}-cargo-index + - name: normalize versions + run: node scripts/normalize-version-bump.js + - name: Setup toolchain run: ${{ matrix.settings.setup }} if: ${{ matrix.settings.setup }} @@ -1351,7 +1351,7 @@ jobs: if: ${{ matrix.settings.docker }} with: image: ${{ matrix.settings.docker }} - options: -e RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -e NAPI_CLI_VERSION=${{ env.NAPI_CLI_VERSION }} -e TURBO_VERSION=${{ env.TURBO_VERSION }} -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build + options: -e RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -e NAPI_CLI_VERSION=${{ env.NAPI_CLI_VERSION }} -e TURBO_VERSION=${{ env.TURBO_VERSION }} -e TURBO_TEAM=vercel -e TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} -e TURBO_REMOTE_ONLY=true -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build run: ${{ matrix.settings.build }} - name: 'Build' @@ -1370,21 +1370,48 @@ jobs: needs: build name: stable - x86_64-unknown-freebsd - node@16 runs-on: macos-12 + env: + TURBO_TEAM: 'vercel' + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_REMOTE_ONLY: 'true' steps: - name: tune mac network run: sudo sysctl -w net.link.generic.system.hwcksum_tx=0 && sudo sysctl -w net.link.generic.system.hwcksum_rx=0 - uses: actions/checkout@v3 + + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version: 16 + check-latest: true + + - run: npm i -g turbo@${{ env.TURBO_VERSION }} pnpm@${PNPM_VERSION} + - name: Delete useless files run: | rm -rf bench rm -rf docs rm -rf errors rm -rf examples - rm -rf scripts rm -rf test + + - run: node scripts/normalize-version-bump.js + + # attempt pulling existing turbo cache, this must be done + # outside of the freeBSD vm as there isn't a turbo build + # for freeBSD currently + - run: node ./scripts/pull-freebsd-cache.js + + - name: check build exists + run: if [ -f packages/next-swc/native/next-swc.freebsd-x64.node ]; then echo "::set-output name=BUILD_EXISTS::yes"; else echo "::set-output name=BUILD_EXISTS::no"; fi + id: build-exists + + - run: echo ${{steps.build-exists.outputs.BUILD_EXISTS}} + - name: Build id: build - uses: vmactions/freebsd-vm@v0.3.0 + uses: vmactions/freebsd-vm@v0 + if: ${{ steps.build-exists.outputs.BUILD_EXISTS == 'no' }} env: DEBUG: napi:* RUSTUP_HOME: /usr/local/rustup @@ -1393,7 +1420,7 @@ jobs: # Disable LTO, or the lld may crash with OOM CARGO_PROFILE_RELEASE_LTO: false with: - envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION TURBO_VERSION RUST_TOOLCHAIN PNPM_VERSION + envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION RUST_TOOLCHAIN PNPM_VERSION usesh: true mem: 6000 prepare: | @@ -1417,6 +1444,11 @@ jobs: pnpm --filter=@next/swc run build-native-no-plugin --platform --release --target x86_64-unknown-freebsd rm -rf node_modules rm -rf packages/next-swc/target + + - name: cache build + if: ${{ steps.build-exists.outputs.BUILD_EXISTS == 'no' }} + run: pnpm turbo run cache-build-native -- --platform --release --target x86_64-unknown-freebsd + - name: Upload artifact uses: actions/upload-artifact@v3 with: @@ -1431,6 +1463,10 @@ jobs: matrix: target: [web, nodejs] runs-on: macos-latest + env: + TURBO_TEAM: 'vercel' + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_REMOTE_ONLY: 'true' steps: - uses: actions/cache@v3 id: restore-build @@ -1454,22 +1490,14 @@ jobs: - run: npm i -g turbo@${{ env.TURBO_VERSION }} pnpm@${PNPM_VERSION} - - name: Turbo cache - id: turbo-cache - uses: actions/cache@v3 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ matrix.target }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ matrix.target }}- - turbo-${{ github.job }}-${{ matrix.target }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}- - turbo-${{ github.job }}-${{ matrix.target }}-canary-${{ needs.build.outputs.weekNum }}- - - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: normalize versions + run: node scripts/normalize-version-bump.js + - name: Build - run: turbo run build-wasm --cache-dir=".turbo" -- --target ${{ matrix.target }} + run: turbo run build-wasm -- --target ${{ matrix.target }} - name: Add target to folder name run: '[[ -d "packages/next-swc/crates/wasm/pkg" ]] && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-${{ matrix.target }} || ls packages/next-swc/crates/wasm' @@ -1513,7 +1541,6 @@ jobs: id: turbo-cache uses: actions/cache@v3 if: ${{needs.build.outputs.docsChange == 'nope'}} - with: path: .turbo key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ steps.get-week.outputs.WEEK }}-${{ github.sha }} @@ -1526,6 +1553,9 @@ jobs: if: ${{needs.build.outputs.docsChange == 'nope'}} run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: normalize versions + run: node scripts/normalize-version-bump.js + - name: Build if: ${{needs.build.outputs.docsChange == 'nope'}} run: turbo run build-wasm --cache-dir=".turbo" -- --target nodejs --dev diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index d6aa012d947ebd8..eb097c192c5ab3b 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -5,7 +5,8 @@ "scripts": { "build-native": "napi build --platform -p next-swc-napi --cargo-name next_swc_napi --features plugin --js false native", "build-native-no-plugin": "napi build --platform -p next-swc-napi --cargo-name next_swc_napi --js false native", - "build-wasm": "wasm-pack build crates/wasm --scope=next" + "build-wasm": "wasm-pack build crates/wasm --scope=next", + "cache-build-native": "echo $(ls native)" }, "napi": { "name": "next-swc", diff --git a/scripts/normalize-version-bump.js b/scripts/normalize-version-bump.js new file mode 100755 index 000000000000000..df455398a3710f3 --- /dev/null +++ b/scripts/normalize-version-bump.js @@ -0,0 +1,66 @@ +#!/usr/bin/env node +// @ts-check + +/* + This prevents busting the turbo cache un-necessarily due + to bumping the version in the repo's package.json files +*/ +const path = require('path') +const fs = require('fs/promises') + +const cwd = process.cwd() +const NORMALIZED_VERSION = '0.0.0' + +const readJson = async (filePath) => + JSON.parse(await fs.readFile(filePath, 'utf8')) + +const writeJson = async (filePath, data) => + fs.writeFile(filePath, JSON.stringify(data, null, 2) + '\n') + +;(async function () { + const packages = await fs.readdir(path.join(cwd, 'packages')) + const pkgJsonData = new Map() + const pkgNames = [] + await Promise.all( + packages.map(async (pkgDir) => { + const data = await readJson( + path.join(cwd, 'packages', pkgDir, 'package.json') + ) + pkgNames.push(data.name) + pkgJsonData.set(pkgDir, data) + }) + ) + const normalizeVersions = async (filePath, data) => { + data = data || (await readJson(filePath)) + const version = data.version + + if (version) { + data.version = NORMALIZED_VERSION + const normalizeEntry = (type, key) => { + const pkgVersion = data[type][key] + + if (pkgNames.includes(key) && pkgVersion === version) { + data[type][key] = NORMALIZED_VERSION + } + } + for (const key of Object.keys(data.dependencies || {})) { + normalizeEntry('dependencies', key) + } + for (const key of Object.keys(data.devDependencies || {})) { + normalizeEntry('devDependencies', key) + } + await writeJson(filePath, data) + } + } + await Promise.all( + packages.map((pkgDir) => + normalizeVersions( + path.join(cwd, 'packages', pkgDir, 'package.json'), + pkgJsonData.get(pkgDir) + ) + ) + ) + await normalizeVersions(path.join(cwd, 'lerna.json')) + await fs.unlink(path.join(cwd, 'pnpm-lock.yaml')) + await fs.writeFile(path.join(cwd, 'pnpm-lock.yaml'), '') +})() diff --git a/scripts/pull-freebsd-cache.js b/scripts/pull-freebsd-cache.js new file mode 100755 index 000000000000000..8e26caaca230cb2 --- /dev/null +++ b/scripts/pull-freebsd-cache.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node +// @ts-check + +const { execSync } = require('child_process') + +;(async function () { + const turboResult = execSync( + `pnpm turbo run cache-build-native --dry=json -- --platform --release --target x86_64-unknown-freebsd` + ).toString() + + const turboData = JSON.parse(turboResult) + + const task = turboData.tasks.find((t) => t.command !== '') + + if (!task) { + console.warn(`Failed to find related turbo task`, turboResult) + return + } + + // pull cache if it was available + if (task.cacheState.local || task.cacheState.remote) { + const pullResult = execSync( + `pnpm turbo run cache-build-native -- --platform --release --target x86_64-unknown-freebsd` + ).toString() + console.log(pullResult) + } else { + console.warn(`No turbo cache was available, continuing...`) + console.warn(task) + } +})() diff --git a/turbo.json b/turbo.json index 2b9677cb6bea5c4..8b9d0365efa817d 100644 --- a/turbo.json +++ b/turbo.json @@ -13,6 +13,10 @@ "dependsOn": ["^build-wasm"], "outputs": ["crates/wasm/pkg/*"] }, + "cache-build-native": { + "dependsOn": ["^cache-build-native"], + "outputs": ["native/*.node"] + }, "build": { "dependsOn": ["^build"], "outputs": ["dist/**"]