review feedback

microsoft · Apr 25, 2024 · 845da1a · 845da1a
1 parent c97db32
commit 845da1a
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 116 deletions.
diff --git a/.github/actions/run-test/action.yml b/.github/actions/run-test/action.yml
@@ -7,16 +7,16 @@ inputs:
   node-version:
     description: Node.js version to use
     required: true
-  browser-to-install:
+  browsers-to-install:
     description: Browser to install
     required: false
     default: ''
-  is-linux:
-    description: 'Is using Linux'
-    required: true
   check-coverage:
     description: 'Validate coverage'
     required: false
+  bot-name:
+    description: 'Bot name'
+    required: true
   flakiness-client-id:
     description: 'Azure Flakiness Dashboard Client ID'
     required: false
@@ -33,6 +33,14 @@ runs:
     - uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
+    # https://github.com/actions/runner-images/issues/9330
+    - name: Allow microphone access to all apps (macOS 14)
+      shell: bash
+      run: |
+        if [[ "$(uname)" == "Darwin" && "$(sw_vers -productVersion | cut -d. -f1)" == "14" ]]; then
+          echo "Allowing microphone access to all apps"
+          sqlite3 $HOME/Library/Application\ Support/com.apple.TCC/TCC.db "INSERT OR IGNORE INTO access VALUES ('kTCCServiceMicrophone','/usr/local/opt/runner/provisioner/provisioner',1,2,4,1,NULL,NULL,0,'UNUSED',NULL,0,1687786159,NULL,NULL,'UNUSED',1687786159);"
+        fi
     - run: npm ci
       shell: bash
       env:
@@ -42,9 +50,16 @@ runs:
       shell: bash
     - run: npx playwright install --with-deps ${{ inputs.install-browser }}
       shell: bash
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- ${{ inputs.command }}
+    - name: Run tests
+      run: |
+        if [[ "$(uname)" == "Linux" ]]; then
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- ${{ inputs.command }}
+        else
+          ${{ inputs.command }}
+        fi
       shell: bash
-      if: ${{ inputs.is-linux }}
+      env:
+        PWTEST_BOT_NAME: ${{ inputs.bot-name }}
     - run: node tests/config/checkCoverage.js ${{ inputs.check-coverage }}
       shell: bash
       if: ${{ inputs.check-coverage }}
@@ -63,4 +78,4 @@ runs:
       uses: ./.github/actions/upload-blob-report
       with:
         report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
+        job_name: ${{ inputs.bot-name }}
diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml
@@ -42,8 +42,6 @@ jobs:
             node-version: 20
             browser: chromium
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
@@ -52,10 +50,10 @@ jobs:
     - uses: ./.github/actions/run-test
       with:
         node-version: ${{ matrix.node-version }}
-        is-linux: true
         validate-coverage: ${{ matrix.browser }}
-        browser-to-install: ${{ matrix.browser }}
+        browsers-to-install: ${{ matrix.browser }}
         command: npm run test -- --project=${{ matrix.browser }}-*
+        bot-name: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}"
         flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
         flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
         flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
@@ -68,8 +66,6 @@ jobs:
       matrix:
         os: [ubuntu-20.04]
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.os }}-chromium-tip-of-tree"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
@@ -78,9 +74,9 @@ jobs:
     - uses: ./.github/actions/run-test
       with:
         node-version: 18
-        is-linux: true
-        browser-to-install: chromium-tip-of-tree
+        browsers-to-install: chromium-tip-of-tree
         command: npm run test -- --project=chromium-*
+        bot-name: "${{ matrix.os }}-chromium-tip-of-tree"
         flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
         flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
         flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}

diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml
@@ -33,22 +33,18 @@ jobs:
         browser: [chromium, firefox, webkit]
         os: [ubuntu-20.04]
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}"
     steps:
     - uses: actions/checkout@v4
     - uses: ./.github/actions/run-test
       with:
         node-version: 18
-        is-linux: true
         check-coverage: ${{ matrix.browser }}
-        browser-to-install: ${{ matrix.browser }} chromium
+        browsers-to-install: ${{ matrix.browser }} chromium
         command: npm run test -- --project=${{ matrix.browser }}-*
+        bot-name: "${{ matrix.browser }}-${{ matrix.os }}"
         flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
         flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
         flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-      env:
-        PWTEST_BOT_NAME: ${{ env.PWTEST_BOT_NAME }}
 
   test_mac:
     name: ${{ matrix.os }} (${{ matrix.browser }})
@@ -59,40 +55,17 @@ jobs:
         os: [macos-12, macos-13, macos-14]
         browser: [chromium, firefox, webkit]
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}"
     steps:
-    # https://github.com/actions/runner-images/issues/9330
-    - name: Allow microphone access to all apps (macOS 14)
-      if: ${{ matrix.os == 'macos-14' }}
-      run: sqlite3 $HOME/Library/Application\ Support/com.apple.TCC/TCC.db "INSERT OR IGNORE INTO access VALUES ('kTCCServiceMicrophone','/usr/local/opt/runner/provisioner/provisioner',1,2,4,1,NULL,NULL,0,'UNUSED',NULL,0,1687786159,NULL,NULL,'UNUSED',1687786159);"
     - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
+    - uses: ./.github/actions/run-test
       with:
         node-version: 18
-    - run: npm ci
-      env:
-        DEBUG: pw:install
-        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-    - run: npm run build
-    - run: npx playwright install --with-deps ${{ matrix.browser }} chromium
-    - run: npm run test -- --project=${{ matrix.browser }}-*
-    - name: Azure Login
-      uses: azure/login@v2
-      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
-      with:
-        client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
-      if: ${{ !cancelled() }}
-      shell: bash
-    - name: Upload blob report
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/upload-blob-report
-      with:
-        report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
+        browsers-to-install: ${{ matrix.browser }} chromium
+        command: npm run test -- --project=${{ matrix.browser }}-*
+        bot-name: "${{ matrix.browser }}-${{ matrix.os }}"
+        flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
+        flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
+        flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
 
   test_win:
     name: "Windows"
@@ -102,41 +75,17 @@ jobs:
       matrix:
         browser: [chromium, firefox, webkit]
     runs-on: windows-latest
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.browser }}-windows-latest"
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
+    - uses: ./.github/actions/run-test
       with:
         node-version: 18
-    - run: npm ci
-      env:
-        DEBUG: pw:install
-        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-    - run: npm run build
-    - run: npx playwright install --with-deps ${{ matrix.browser }} chromium
-    - run: npm run test -- --project=${{ matrix.browser }}-* --workers=1
-      if: matrix.browser == 'firefox'
-      shell: bash
-    - run: npm run test -- --project=${{ matrix.browser }}-*
-      if: matrix.browser != 'firefox'
-      shell: bash
-    - name: Azure Login
-      uses: azure/login@v2
-      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
-      with:
-        client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
-      if: ${{ !cancelled() }}
-      shell: bash
-    - name: Upload blob report
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/upload-blob-report
-      with:
-        report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
+        browsers-to-install: ${{ matrix.browser }} chromium
+        command: npm run test -- --project=${{ matrix.browser }}-* ${{ matrix.browser == 'firefox' && '--workers 1' || '' }}
+        bot-name: "${{ matrix.browser }}-windows-latest"
+        flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
+        flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
+        flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
 
   test-package-installations-other-node-versions:
     name: "Installation Test ${{ matrix.os }} (${{ matrix.node_version }})"
@@ -187,43 +136,17 @@ jobs:
         browser: [chromium, firefox, webkit]
         os: [ubuntu-20.04, ubuntu-22.04, macos-14, windows-latest]
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.browser }}-headed-${{ matrix.os }}"
     steps:
-    # https://github.com/actions/runner-images/issues/9330
-    - name: Allow microphone access to all apps (macOS 14)
-      if: ${{ matrix.os == 'macos-14' }}
-      run: sqlite3 $HOME/Library/Application\ Support/com.apple.TCC/TCC.db "INSERT OR IGNORE INTO access VALUES ('kTCCServiceMicrophone','/usr/local/opt/runner/provisioner/provisioner',1,2,4,1,NULL,NULL,0,'UNUSED',NULL,0,1687786159,NULL,NULL,'UNUSED',1687786159);"
     - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
+    - uses: ./.github/actions/run-test
       with:
         node-version: 18
-    - run: npm ci
-      env:
-        DEBUG: pw:install
-        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-    - run: npm run build
-    - run: npx playwright install --with-deps ${{ matrix.browser }} chromium
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}-* --headed
-      if: ${{ !cancelled() && startsWith(matrix.os, 'ubuntu-') }}
-    - run: npm run test -- --project=${{ matrix.browser }}-* --headed
-      if: ${{ !cancelled() && !startsWith(matrix.os, 'ubuntu-') }}
-    - name: Azure Login
-      uses: azure/login@v2
-      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
-      with:
-        client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
-      if: ${{ !cancelled() }}
-      shell: bash
-    - name: Upload blob report
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/upload-blob-report
-      with:
-        report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
+        browsers-to-install: ${{ matrix.browser }} chromium
+        command: npm run test -- --project=${{ matrix.browser }}-* --headed
+        bot-name: "${{ matrix.browser }}-headed-${{ matrix.os }}"
+        flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
+        flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
+        flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
 
   transport_linux:
     name: "Transport"
@@ -1084,7 +1007,6 @@ jobs:
     - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium-*
       env:
         PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW: 1
-    - run: node tests/config/checkCoverage.js chromium
     - name: Azure Login
       uses: azure/login@v2
       if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}