H-1897: Upload benchmark results to S3 #166
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bench | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: hashintel | |
| TURBO_REMOTE_ONLY: true | |
| VAULT_ADDR: ${{ secrets.VAULT_ADDR }} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| unit: ${{ steps.packages.outputs.unit }} | |
| integration: ${{ steps.packages.outputs.integration }} | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install turbo | |
| uses: ./.github/actions/install-turbo | |
| - name: Determine changed packages | |
| id: packages | |
| run: | | |
| UNIT_BENCH_FILTER=$(turbo run bench:unit --dry-run=json --filter '...[HEAD^]' | jq -e '.packages | contains(["//"])' > /dev/null && echo '' || echo '--filter ...[HEAD^]') | |
| UNIT_BENCH_TASKS=$(sh -c "turbo run bench:unit --dry-run=json $UNIT_BENCH_FILTER" | jq -c '.tasks[]') | |
| UNIT_BENCH_PACKAGES=$(echo "$UNIT_BENCH_TASKS" \ | |
| | jq 'select(.task == "bench:unit" and .command != "<NONEXISTENT>")' \ | |
| | jq --compact-output --slurp '{ package: [.[].package] | unique, include: [( .[] | {package: .package, directory: .directory })] | unique }') | |
| INTEGRATION_BENCH_FILTER=$(turbo run bench:integration --dry-run=json --filter '...[HEAD^]' | jq -e '.packages | contains(["//"])' > /dev/null && echo '' || echo '--filter ...[HEAD^]') | |
| INTEGRATION_BENCH_TASKS=$(sh -c "turbo run bench:integration --dry-run=json $INTEGRATION_BENCH_FILTER" | jq -c '.tasks[]') | |
| INTEGRATION_BENCH_PACKAGES=$(echo "$INTEGRATION_BENCH_TASKS" \ | |
| | jq 'select(.task == "bench:integration" and .command != "<NONEXISTENT>")' \ | |
| | jq --compact-output --slurp '{ package: [.[].package] | unique, include: [( .[] | {package: .package, directory: .directory })] | unique }') | |
| set -x | |
| echo "unit=$UNIT_BENCH_PACKAGES" >> $GITHUB_OUTPUT | |
| echo "integration=$INTEGRATION_BENCH_PACKAGES" >> $GITHUB_OUTPUT | |
| unit-benches: | |
| name: Unit | |
| needs: [setup] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| matrix: ${{ fromJSON(needs.setup.outputs.unit) }} | |
| fail-fast: false | |
| if: needs.setup.outputs.unit != '{"package":[],"include":[]}' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install turbo | |
| uses: ./.github/actions/install-turbo | |
| - name: Find bench steps to run | |
| id: benches | |
| run: | | |
| if [[ "${{ github.event_name == 'pull_request' }}" == 'true' ]]; then | |
| echo "create-baseline=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "create-baseline=false" >> $GITHUB_OUTPUT | |
| fi | |
| HAS_RUST=$([[ -f "${{ matrix.directory }}/Cargo.toml" || ${{ matrix.directory }} = "apps/hash-graph" ]] && echo 'true' || echo 'false') | |
| echo "has-rust=$HAS_RUST" >> $GITHUB_OUTPUT | |
| if [[ $HAS_RUST = 'true' ]]; then | |
| if [[ -f "${{ matrix.directory }}/rust-toolchain.toml" ]]; then | |
| RUST_TOOLCHAIN_FILE="${{ matrix.directory }}/rust-toolchain.toml" | |
| else | |
| RUST_TOOLCHAIN_FILE="rust-toolchain.toml" | |
| fi | |
| echo "rust-toolchain=$(yq '.toolchain.channel' $RUST_TOOLCHAIN_FILE)" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Install Rust tools | |
| if: steps.benches.outputs.has-rust == 'true' | |
| uses: taiki-e/install-action@c2927f0c5b5adc6a76bc4a7847bc6e0503754bed # v2.33.22 | |
| with: | |
| tool: just@1.13.0,critcmp@0.1.8 | |
| - name: Install Protobuf | |
| if: steps.benches.outputs.has-rust == 'true' | |
| run: sudo apt install protobuf-compiler | |
| - name: Checkout base branch | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: | | |
| git fetch origin "$GITHUB_BASE_REF" | |
| git checkout FETCH_HEAD | |
| - name: Install Rust toolchain | |
| if: steps.benches.outputs.has-rust == 'true' && steps.benches.outputs.create-baseline == 'true' | |
| uses: ./.github/actions/install-rust-toolchain | |
| with: | |
| toolchain: ${{ steps.benches.outputs.rust-toolchain }} | |
| working-directory: ${{ matrix.directory }} | |
| - name: Prune repository | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.package }} | |
| @rust/repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Cache Rust dependencies | |
| if: steps.benches.outputs.has-rust == 'true' | |
| uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 | |
| with: | |
| workspaces: ${{ matrix.directory }} | |
| save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }} | |
| - name: Run base benches | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: turbo run bench:unit --filter "${{ matrix.package }}" | |
| - name: Checkout head branch | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: | | |
| git fetch origin "$GITHUB_HEAD_REF" | |
| git reset --hard FETCH_HEAD | |
| - name: Install Rust toolchain | |
| if: steps.benches.outputs.has-rust == 'true' | |
| uses: ./.github/actions/install-rust-toolchain | |
| with: | |
| toolchain: ${{ steps.benches.outputs.rust-toolchain }} | |
| working-directory: ${{ matrix.directory }} | |
| - name: Prune repository | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.package }} | |
| @rust/repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Run head benches | |
| run: turbo run bench:unit --filter "${{ matrix.package }}" | |
| - name: Analyze benchmarks | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: | | |
| TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g') | |
| echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV | |
| echo '<details><summary><h2>${{ matrix.package }} – Units</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| turbo run analyze-benchmarks -- --output "/tmp/benches.md" | |
| cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| - name: Authenticate Vault | |
| id: secrets | |
| uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 | |
| with: | |
| exportToken: true | |
| url: ${{ env.VAULT_ADDR }} | |
| method: jwt | |
| role: prod | |
| secrets: | | |
| aws/creds/prod-deploy access_key | AWS_ACCESS_KEY_ID ; | |
| aws/creds/prod-deploy secret_key | AWS_SECRET_ACCESS_KEY ; | |
| aws/creds/prod-deploy security_token | AWS_SESSION_TOKEN | |
| - name: Upload benchmark results | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| run: turbo run upload-benchmarks | |
| - name: Upload benchmark summary | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 | |
| with: | |
| name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }} | |
| path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md | |
| integration-benches: | |
| name: Integration | |
| needs: [setup] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| matrix: ${{ fromJSON(needs.setup.outputs.integration) }} | |
| fail-fast: false | |
| if: needs.setup.outputs.integration != '{"package":[],"include":[]}' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install turbo | |
| uses: ./.github/actions/install-turbo | |
| - name: Find bench steps to run | |
| id: benches | |
| run: | | |
| if [[ "${{ github.event_name == 'pull_request' }}" == 'true' ]]; then | |
| echo "create-baseline=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "create-baseline=false" >> $GITHUB_OUTPUT | |
| fi | |
| HAS_RUST=$([[ -f "${{ matrix.directory }}/Cargo.toml" || ${{ matrix.directory }} = "apps/hash-graph" ]] && echo 'true' || echo 'false') | |
| echo "has-rust=$HAS_RUST" >> $GITHUB_OUTPUT | |
| if [[ $HAS_RUST = 'true' ]]; then | |
| if [[ -f "${{ matrix.directory }}/rust-toolchain.toml" ]]; then | |
| RUST_TOOLCHAIN_FILE="${{ matrix.directory }}/rust-toolchain.toml" | |
| else | |
| RUST_TOOLCHAIN_FILE="rust-toolchain.toml" | |
| fi | |
| echo "rust-toolchain=$(yq '.toolchain.channel' $RUST_TOOLCHAIN_FILE)" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Install Rust tools | |
| if: steps.benches.outputs.has-rust == 'true' | |
| uses: taiki-e/install-action@c2927f0c5b5adc6a76bc4a7847bc6e0503754bed # v2.33.22 | |
| with: | |
| tool: just@1.13.0,critcmp@0.1.8 | |
| - name: Install Protobuf | |
| if: steps.benches.outputs.has-rust == 'true' | |
| run: sudo apt install protobuf-compiler | |
| - name: Checkout base branch | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: | | |
| git fetch origin "$GITHUB_BASE_REF" | |
| git checkout FETCH_HEAD | |
| - name: Install Rust toolchain | |
| if: steps.benches.outputs.has-rust == 'true' && steps.benches.outputs.create-baseline == 'true' | |
| uses: ./.github/actions/install-rust-toolchain | |
| with: | |
| toolchain: ${{ steps.benches.outputs.rust-toolchain }} | |
| working-directory: ${{ matrix.directory }} | |
| - name: Prune repository | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.package }} | |
| @rust/repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Cache Rust dependencies | |
| if: steps.benches.outputs.has-rust == 'true' | |
| uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 | |
| with: | |
| workspaces: ${{ matrix.directory }} | |
| save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }} | |
| - name: Launch external services | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: | | |
| turbo codegen --filter '@apps/hash-external-services' | |
| SERVICES="hydra kratos redis spicedb" | |
| echo "Running services: $SERVICES" | |
| yarn workspace @apps/hash-external-services deploy:test up $SERVICES --wait | |
| - name: Run base benches | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: turbo run bench:integration --filter "${{ matrix.package }}" | |
| - name: Tear down external services | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: yarn workspace @apps/hash-external-services deploy:test down | |
| - name: Checkout head branch | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: | | |
| git fetch origin "$GITHUB_HEAD_REF" | |
| git reset --hard FETCH_HEAD | |
| - name: Install Rust toolchain | |
| if: steps.benches.outputs.has-rust == 'true' | |
| uses: ./.github/actions/install-rust-toolchain | |
| with: | |
| toolchain: ${{ steps.benches.outputs.rust-toolchain }} | |
| working-directory: ${{ matrix.directory }} | |
| - name: Prune repository | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.package }} | |
| @rust/repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Launch external services | |
| run: | | |
| turbo codegen --filter '@apps/hash-external-services' | |
| SERVICES="hydra kratos redis spicedb" | |
| echo "Running services: $SERVICES" | |
| yarn workspace @apps/hash-external-services deploy:test up $SERVICES --wait | |
| - name: Run head benches | |
| run: turbo run bench:integration --filter "${{ matrix.package }}" | |
| - name: Analyze benchmarks | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| run: | | |
| TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g') | |
| echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV | |
| echo '<details><summary><h2>${{ matrix.package }} – Integrations</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| turbo run analyze-benchmarks -- --output "/tmp/benches.md" | |
| cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| - name: Authenticate Vault | |
| id: secrets | |
| uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 | |
| with: | |
| exportToken: true | |
| url: ${{ env.VAULT_ADDR }} | |
| method: jwt | |
| role: prod | |
| secrets: | | |
| aws/creds/prod-deploy access_key | AWS_ACCESS_KEY_ID ; | |
| aws/creds/prod-deploy secret_key | AWS_SECRET_ACCESS_KEY ; | |
| aws/creds/prod-deploy security_token | AWS_SESSION_TOKEN | |
| - name: Upload benchmark results | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| run: turbo run upload-benchmarks | |
| - name: Upload benchmark summary | |
| if: steps.benches.outputs.create-baseline == 'true' | |
| uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 | |
| with: | |
| name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }} | |
| path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md | |
| - name: Show container logs | |
| if: ${{ success() || failure() }} | |
| run: yarn workspace @apps/hash-external-services deploy logs --timestamps | |
| passed: | |
| name: Benches passed | |
| needs: [setup, unit-benches, integration-benches] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Check setup script | |
| run: | | |
| [[ ${{ needs.setup.result }} = success ]] | |
| - name: Check unit benches | |
| run: | | |
| [[ ${{ needs.unit-benches.result }} =~ success|skipped ]] | |
| - name: Check integration benches | |
| run: | | |
| [[ ${{ needs.integration-benches.result }} =~ success|skipped ]] | |
| - name: Notify Slack on failure | |
| uses: rtCamp/action-slack-notify@19c86e40d4bc3be986b18f1459d4549536887065 | |
| if: ${{ failure() && github.event_name == 'merge_group' }} | |
| env: | |
| SLACK_LINK_NAMES: true | |
| SLACK_MESSAGE: "At least one bench job failed for a Pull Request in the Merge Queue failed <@U0143NL4GMP> <@U027NPY8Y3X> <@U02NLJY0FGX>" | |
| SLACK_TITLE: Tests failed | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_USERNAME: GitHub | |
| VAULT_ADDR: "" | |
| VAULT_TOKEN: "" | |
| - name: Checkout | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| - name: Download reports' artifacts | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 | |
| with: | |
| path: downloads | |
| - name: Create comment | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| run: | | |
| echo "## Benchmark results" > /tmp/summary.md | |
| cat downloads/*/*.md >> /tmp/summary.md | |
| - name: Post comment | |
| uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| with: | |
| message-path: /tmp/summary.md | |
| message-id: benchmark-results | |
| refresh-message-position: true |