diff --git a/.github/workflows/performance-report.yml b/.github/workflows/performance-report.yml new file mode 100644 index 00000000000..6e4a8bdf611 --- /dev/null +++ b/.github/workflows/performance-report.yml @@ -0,0 +1,151 @@ +name: Performance Report +env: + BUILD_BOOTSTRAP_CJS: mv dist dist-build && node dist-build/bin/rollup --config rollup.config.ts --configPlugin typescript --configTest --forceExit && rm -rf dist-build + +on: + pull_request: + types: + - synchronize + - opened + - reopened + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build-artefacts: + strategy: + matrix: + settings: + - name: current + ref: refs/pull/${{ github.event.number }}/merge + - name: previous + ref: ${{github.event.pull_request.base.ref}} + name: Build ${{matrix.settings.name}} artefact + runs-on: ubuntu-latest + steps: + - name: Checkout Commit + uses: actions/checkout@v4 + with: + ref: ${{matrix.settings.ref}} + - name: Install Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly-2023-10-05 + targets: x86_64-unknown-linux-gnu + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + .cargo-cache + rust/target/ + key: cargo-cache-${{ hashFiles('rust/Cargo.lock') }} + restore-keys: cargo-cache + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Cache Node Modules + id: cache-node-modules + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: npm ci --ignore-scripts + - name: Build artefacts 123 + run: npm exec -- concurrently -c green,blue 'npm:build:napi -- --release' 'npm:build:cjs' && npm run build:copy-native && ${{env.BUILD_BOOTSTRAP_CJS}} && npm run build:copy-native + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.settings.name }} + path: dist/ + if-no-files-found: error + + report: + needs: build-artefacts + permissions: + pull-requests: write + runs-on: ubuntu-latest + name: Report Performance + steps: + - name: Checkout Commit + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.number }}/merge + - name: Install Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly-2023-10-05 + targets: x86_64-unknown-linux-gnu + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + .cargo-cache + rust/target/ + key: cargo-cache-${{ hashFiles('rust/Cargo.lock') }} + restore-keys: cargo-cache + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Cache Node Modules + id: cache-node-modules + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: npm ci --ignore-scripts + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: _benchmark + - name: Change rollup import in internal benchmark + run: | + echo "export { rollup as previousRollup, VERSION as previousVersion } from '../../_benchmark/previous/rollup.js';" > ./scripts/perf-report/rollup-artefacts.js + echo "export { rollup as newRollup } from '../../_benchmark/current/rollup.js';" >> ./scripts/perf-report/rollup-artefacts.js + - name: Run internal benchmark + run: node --expose-gc scripts/perf-report/index.js + - name: Install benchmark tool + run: cargo install --locked hyperfine + - name: Run Rough benchmark + run: | + hyperfine --warmup 1 --export-markdown _benchmark/rough-report.md --show-output --runs 3 \ + 'node _benchmark/previous/bin/rollup -i ./perf/entry.js -o _benchmark/result/previous.js' \ + 'node _benchmark/current/bin/rollup -i ./perf/entry.js -o _benchmark/result/current.js' + - name: Combine bechmark reports + run: | + echo "# Performance report!" > _benchmark/result.md + echo "## Rough benchmark" >> _benchmark/result.md + cat _benchmark/rough-report.md >> _benchmark/result.md + echo "## Internal benchmark" >> _benchmark/result.md + cat _benchmark/internal-report.md >> _benchmark/result.md + - name: Find Performance report + uses: peter-evans/find-comment@v3 + id: findPerformanceReport + with: + issue-number: ${{ github.event.number }} + comment-author: 'github-actions[bot]' + body-includes: 'Performance report' + - name: Create or update Performance report + uses: peter-evans/create-or-update-comment@v4 + id: createOrUpdatePerformanceReport + with: + comment-id: ${{ steps.findPerformanceReport.outputs.comment-id }} + issue-number: ${{ github.event.number }} + edit-mode: replace + body-path: _benchmark/result.md diff --git a/package.json b/package.json index 2751cc14c6c..62cdaaf6b5d 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "lint:markdown:nofix": "prettier --check \"**/*.md\"", "lint:rust": "cd rust && cargo fmt && cargo clippy --fix --allow-dirty", "lint:rust:nofix": "cd rust && cargo fmt --check && cargo clippy", - "perf": "npm run build && node --expose-gc scripts/perf.js", + "perf": "npm run build && node --expose-gc scripts/perf-report/index.js", "prepare": "husky && node scripts/check-release.js || npm run build:prepare", "prepublishOnly": "node scripts/check-release.js && node scripts/prepublish.js", "postpublish": "node scripts/postpublish.js", diff --git a/scripts/perf.js b/scripts/perf-report/index.js similarity index 75% rename from scripts/perf.js rename to scripts/perf-report/index.js index e8aba413bf6..f06e723429a 100644 --- a/scripts/perf.js +++ b/scripts/perf-report/index.js @@ -5,10 +5,9 @@ import { chdir } from 'node:process'; import { fileURLToPath } from 'node:url'; import { createColors } from 'colorette'; import prettyBytes from 'pretty-bytes'; -import { rollup as previousRollup, VERSION as previousVersion } from 'rollup'; -// eslint-disable-next-line import/no-unresolved -import { rollup as newRollup } from '../dist/rollup.js'; -import { runWithEcho } from './helpers.js'; +import { runWithEcho } from '../helpers.js'; +import reportCollector from './report-collector.js'; +import { newRollup, previousRollup, previousVersion } from './rollup-artefacts.js'; /** * @typedef {Record} CollectedTimings @@ -18,7 +17,7 @@ import { runWithEcho } from './helpers.js'; * @typedef {Record} AccumulatedTimings */ -const PERF_DIRECTORY = new URL('../perf/', import.meta.url); +const PERF_DIRECTORY = new URL('../../perf/', import.meta.url); const ENTRY = new URL('entry.js', PERF_DIRECTORY); const THREEJS_COPIES = 10; const { bold, underline, cyan, red, green } = createColors(); @@ -91,10 +90,12 @@ async function calculatePrintAndPersistTimings() { ); clearLines(numberOfLinesToClear); } + reportCollector.startRecord(); printMeasurements( getAverage(accumulatedNewTimings, RUNS_TO_AVERAGE), getAverage(accumulatedPreviousTimings, RUNS_TO_AVERAGE) ); + await reportCollector.outputMsg(); } /** @@ -172,9 +173,16 @@ function getSingleAverage(times, runs, discarded) { * @return {number} */ function printMeasurements(newAverage, previousAverage, filter = /.*/) { - const printedLabels = Object.keys(newAverage).filter(label => filter.test(label)); - console.info(''); - for (const label of printedLabels) { + const newPrintedLabels = Object.keys(newAverage).filter(predicateLabel); + const previousPrintedLabels = Object.keys(previousAverage).filter(predicateLabel); + + const newTreeShakings = newPrintedLabels.filter(isTreeShakingLabel); + const oldTreeShakings = previousPrintedLabels.filter(isTreeShakingLabel); + + const addedTreeShaking = newTreeShakings.length - oldTreeShakings.length; + let treeShakingCount = 0; + + for (const label of newPrintedLabels) { /** * @type {function(string): string} */ @@ -185,16 +193,54 @@ function printMeasurements(newAverage, previousAverage, filter = /.*/) { color = underline; } } - console.info( - color( - `${label}: ${getFormattedTime( - newAverage[label].time, - previousAverage[label]?.time - )}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}` - ) - ); + const texts = []; + if (isTreeShakingLabel(label)) { + treeShakingCount++; + if (addedTreeShaking < 0 && treeShakingCount === newTreeShakings.length) { + texts.push(generateSingleReport(label)); + for (const label of oldTreeShakings.slice(addedTreeShaking)) { + const { time, memory } = previousAverage[label]; + texts.push(`${label}: ${time.toFixed(0)}ms, ${prettyBytes(memory)}, removed stage`); + } + } else if (addedTreeShaking > 0 && treeShakingCount > oldTreeShakings.length) { + texts.push(generateSingleReport(label, ', new stage')); + } else { + texts.push(generateSingleReport(label)); + } + } else { + texts.push(generateSingleReport(label)); + } + for (const text of texts) { + reportCollector.push(text); + console.info(color(text)); + } + } + return Math.max(newPrintedLabels.length, previousPrintedLabels.length) + 2; + + /** + * @param {string} label + */ + function predicateLabel(label) { + return filter.test(label); + } + + /** + * @param {string} label + * @param {string} addon + */ + function generateSingleReport(label, addon = '') { + return `${label}: ${getFormattedTime( + newAverage[label].time, + previousAverage[label]?.time + )}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}${addon}`; } - return printedLabels.length + 2; +} + +/** + * @param {string} label + */ +function isTreeShakingLabel(label) { + return label.startsWith('treeshaking pass'); } /** diff --git a/scripts/perf-report/report-collector.js b/scripts/perf-report/report-collector.js new file mode 100644 index 00000000000..5201c038880 --- /dev/null +++ b/scripts/perf-report/report-collector.js @@ -0,0 +1,47 @@ +import { writeFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; + +export default new (class ReportCollector { + /** + * @type {string[]} + */ + #messageList = []; + #isRecording = false; + startRecord() { + this.#isRecording = true; + } + /** + * @param {string} message + */ + push(message) { + if (!this.#isRecording) return; + if (message.startsWith('#')) { + message = '##' + message; + } + this.#messageList.push(message); + } + outputMsg() { + if (process.env.CI) { + return writeFile( + fileURLToPath(new URL('../../_benchmark/internal-report.md', import.meta.url)), + removeAnsiStyles(this.#messageList.join('\n')) + ); + } + } +})(); + +/** + * @param {string} text + * @returns {string} + */ +function removeAnsiStyles(text) { + const ansiRegex = new RegExp( + [ + '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\\u0007)', + '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))' + ].join('|'), + 'g' + ); + + return text.replace(ansiRegex, ''); +} diff --git a/scripts/perf-report/rollup-artefacts.js b/scripts/perf-report/rollup-artefacts.js new file mode 100644 index 00000000000..936e44ce426 --- /dev/null +++ b/scripts/perf-report/rollup-artefacts.js @@ -0,0 +1,3 @@ +export { rollup as previousRollup, VERSION as previousVersion } from 'rollup'; +// eslint-disable-next-line import/no-unresolved +export { rollup as newRollup } from '../../dist/rollup.js';