review feedback

microsoft · Apr 25, 2024 · 099cc34 · 099cc34
1 parent c97db32
commit 099cc34
Show file tree

Hide file tree

Showing 3 changed files with 236 additions and 815 deletions.
diff --git a/.github/actions/run-test/action.yml b/.github/actions/run-test/action.yml
@@ -6,17 +6,18 @@ inputs:
     required: true
   node-version:
     description: Node.js version to use
-    required: true
-  browser-to-install:
-    description: Browser to install
+    required: false
+    default: '18'
+  browsers-to-install:
+    description: Browser to install. Default is all browsers.
     required: false
     default: ''
-  is-linux:
-    description: 'Is using Linux'
-    required: true
   check-coverage:
     description: 'Validate coverage'
     required: false
+  bot-name:
+    description: 'Bot name'
+    required: true
   flakiness-client-id:
     description: 'Azure Flakiness Dashboard Client ID'
     required: false
@@ -33,18 +34,33 @@ runs:
     - uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
+    # https://github.com/actions/runner-images/issues/9330
+    - name: Allow microphone access to all apps (macOS 14)
+      shell: bash
+      run: |
+        if [[ "$(uname)" == "Darwin" && "$(sw_vers -productVersion | cut -d. -f1)" == "14" ]]; then
+          echo "Allowing microphone access to all apps"
+          sqlite3 $HOME/Library/Application\ Support/com.apple.TCC/TCC.db "INSERT OR IGNORE INTO access VALUES ('kTCCServiceMicrophone','/usr/local/opt/runner/provisioner/provisioner',1,2,4,1,NULL,NULL,0,'UNUSED',NULL,0,1687786159,NULL,NULL,'UNUSED',1687786159);"
+        fi
     - run: npm ci
       shell: bash
       env:
         DEBUG: pw:install
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
     - run: npm run build
       shell: bash
-    - run: npx playwright install --with-deps ${{ inputs.install-browser }}
+    - run: npx playwright install --with-deps ${{ inputs.browsers-to-install }}
       shell: bash
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- ${{ inputs.command }}
+    - name: Run tests
+      run: |
+        if [[ "$(uname)" == "Linux" ]]; then
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- ${{ inputs.command }}
+        else
+          ${{ inputs.command }}
+        fi
       shell: bash
-      if: ${{ inputs.is-linux }}
+      env:
+        PWTEST_BOT_NAME: ${{ inputs.bot-name }}
     - run: node tests/config/checkCoverage.js ${{ inputs.check-coverage }}
       shell: bash
       if: ${{ inputs.check-coverage }}
@@ -63,4 +79,4 @@ runs:
       uses: ./.github/actions/upload-blob-report
       with:
         report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
+        job_name: ${{ inputs.bot-name }}
diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml
@@ -42,8 +42,6 @@ jobs:
             node-version: 20
             browser: chromium
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
@@ -52,10 +50,10 @@ jobs:
     - uses: ./.github/actions/run-test
       with:
         node-version: ${{ matrix.node-version }}
-        is-linux: true
         validate-coverage: ${{ matrix.browser }}
-        browser-to-install: ${{ matrix.browser }}
+        browsers-to-install: ${{ matrix.browser }}
         command: npm run test -- --project=${{ matrix.browser }}-*
+        bot-name: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}"
         flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
         flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
         flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
@@ -68,25 +66,21 @@ jobs:
       matrix:
         os: [ubuntu-20.04]
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.os }}-chromium-tip-of-tree"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
     steps:
     - uses: actions/checkout@v4
     - uses: ./.github/actions/run-test
       with:
-        node-version: 18
-        is-linux: true
-        browser-to-install: chromium-tip-of-tree
+        browsers-to-install: chromium-tip-of-tree
         command: npm run test -- --project=chromium-*
+        bot-name: "${{ matrix.os }}-chromium-tip-of-tree"
         flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
         flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
         flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
       env:
         PWTEST_CHANNEL: chromium-tip-of-tree
-        PWTEST_BOT_NAME: "${{ matrix.os }}-chromium-tip-of-tree"
 
   test_test_runner:
     name: Test Runner
@@ -116,41 +110,21 @@ jobs:
             shardIndex: 2
             shardTotal: 2
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.os }}-node${{ matrix.node-version }}-${{ matrix.shardIndex }}"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
+    - uses: ./.github/actions/run-test
       with:
         node-version: ${{matrix.node-version}}
-    - run: npm ci
+        command: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        bot-name: "${{ matrix.os }}-node${{ matrix.node-version }}-${{ matrix.shardIndex }}"
+        flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
+        flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
+        flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
       env:
-        DEBUG: pw:install
-    - run: npm run build
-    - run: npx playwright install --with-deps
-    - run: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
-      if: matrix.os != 'ubuntu-latest'
-    - run: xvfb-run npm run ttest --  --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
-      if: matrix.os == 'ubuntu-latest'
-    - name: Azure Login
-      uses: azure/login@v2
-      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
-      with:
-        client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
-      if: ${{ !cancelled() }}
-      shell: bash
-    - name: Upload blob report
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/upload-blob-report
-      with:
-        report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
+        PWTEST_CHANNEL: firefox-beta
 
   test_web_components:
     name: Web Components
@@ -235,40 +209,17 @@ jobs:
         - windows-latest
     runs-on: ${{ matrix.os  }}
     timeout-minutes: 30
-    env:
-      PWTEST_BOT_NAME: "package-installations-${{ matrix.os }}"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
-      with:
-        node-version: 18
-    - run: npm ci
-      env:
-        DEBUG: pw:install
-    - run: npm run build
-    - run: npx playwright install --with-deps
     - run: npm install -g yarn@1
     - run: npm install -g pnpm@8
-    - run: npm run itest
-      if: matrix.os != 'ubuntu-latest'
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run itest
-      if: matrix.os == 'ubuntu-latest'
-    - name: Azure Login
-      uses: azure/login@v2
-      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
-      with:
-        client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
-      if: ${{ !cancelled() }}
-      shell: bash
-    - name: Upload blob report
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/upload-blob-report
+    - uses: ./.github/actions/run-test
       with:
-        report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
+        command: npm run itest
+        bot-name: "package-installations-${{ matrix.os }}"
+        flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
+        flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
+        flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}