Fail CI on new test262 failures (#15091)

* Fail CI on new test262 failures * Update .github/workflows/ci.yml
babel · Oct 29, 2022 · df147ad · Kadircan6666 · Jul 12, 2023 · df147ad
1 parent 19090d4
commit df147ad
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 100 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -21,18 +21,6 @@ aliases:
   - &artifact_babel_min_sourcemap
     path: ~/babel/packages/babel-standalone/babel.min.js.map
 
-  - &test262_workdir
-    working_directory: ~/babel/babel-test262-runner
-
-  - &artifact_test262_tap
-    path: ~/test262.tap
-
-  - &artifact_test262_xunit
-    path: ~/test-results
-
-  - &artifact_test262_diff_tap
-    path: ~/diff.tap
-
 executors:
   node-executor:
     docker:
@@ -57,96 +45,8 @@ jobs:
       - store_artifacts: *artifact_babel_min_sourcemap
       - save_cache: *save-yarn-cache
 
-  test262:
-    executor: node-executor
-    steps:
-      - checkout
-      - run:
-          name: Sync with latest main branch (only on PRs)
-          command: |
-            if [ -n "$CIRCLE_PULL_REQUEST" ]
-            then
-              # $CIRCLE_PR_NUMBER is not always defined:
-              # https://support.circleci.com/hc/en-us/articles/360047521451-Why-is-CIRCLE-PR-NUMBER-empty-
-              git fetch origin refs/pull/${CIRCLE_PULL_REQUEST##*/}/merge
-              git checkout -qf FETCH_HEAD
-            fi
-      - restore_cache: *restore-yarn-cache
-      - run:
-          name: Build Babel
-          command: BABEL_ENV=test make bootstrap
-      - run:
-          name: Setup Test Runner
-          command: |
-            git clone --depth=1 --recurse-submodules https://github.com/babel/babel-test262-runner
-            cd babel-test262-runner
-            npm ci
-            npm i tap-mocha-reporter --save-dev
-            node lib/download-node
-      - run:
-          name: Download main branch Test262 artifact
-          command: node lib/download-main-artifact ~/test262-main.tap
-          <<: *test262_workdir
-      - run:
-          name: Run Test262
-          command: BABEL_PATH=.. node lib/run-tests I_AM_SURE | tee ~/test262.tap
-          <<: *test262_workdir
-      - store_artifacts: *artifact_test262_tap
-      - run:
-          name: Output Test262 results
-          command: |
-            cat ~/test262.tap | $(npm bin)/tap-mocha-reporter spec || true
-          <<: *test262_workdir
-      - run:
-          name: Compare previous main branch & current job results
-          command: |
-            mkdir -p ~/test-results/test262
-            node lib/compare-results ~/test262-main.tap ~/test262.tap | tee ~/diff.tap
-          <<: *test262_workdir
-      - store_artifacts: *artifact_test262_diff_tap
-      - run:
-          name: Output comparison results and report to CircleCI
-          command: |
-            mkdir -p ~/test-results/test262
-            cat ~/diff.tap | $(npm bin)/tap-merge | $(npm bin)/tap-mocha-reporter xunit | tee ~/test-results/test262/results.xml
-          <<: *test262_workdir
-      - store_test_results: *artifact_test262_xunit
-      - save_cache: *save-yarn-cache
-
 workflows:
   version: 2
   build-standalone:
     jobs:
       - build-standalone
-  # test262:
-  #   jobs:
-  #     - test262:
-  #         filters:
-  #           branches:
-  #             only:
-  #               - main
-  #               - next-8-dev
-  #               - next-8-rebased
-  #         context: babel-test262
-  # test262-pr:
-  #   jobs:
-  #     - approve-test262-run:
-  #         type: approval
-  #         filters:
-  #           branches:
-  #             ignore:
-  #               - main
-  #               - master
-  #               - next-8-dev
-  #               - next-8-rebased
-  #     - test262:
-  #         requires:
-  #           - approve-test262-run
-  #         filters:
-  #           branches:
-  #             ignore:
-  #               - main
-  #               - master
-  #               - next-8-dev
-  #               - next-8-rebased
-  #         context: babel-test262
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -633,3 +633,6 @@ jobs:
         run: |
           cd babel-test262-runner
           node lib/compare-results ~/test262-main.tap ../test262.tap | tee ~/diff.tap
+        # Specify bash to enable pipefail, so that "| tee" does not suppress the failure exit code.
+        # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
+        shell: bash