From 254b49e5897b4e493c88ecec5c80abeacda90526 Mon Sep 17 00:00:00 2001
From: Luna Ruan <lunaris.ruan@gmail.com>
Date: Mon, 6 Jun 2022 10:36:58 -0700
Subject: [PATCH] Add snapshot testing on e2e test failure (#24672)

We have a currently unreproducible flaky e2e test. This PR captures snapshots on e2e test failures so we can better debug flaky e2e tests that don't fail locally.
---
 .circleci/config.yml                                       | 2 ++
 .../__tests__/__e2e__/components.test.js                   | 1 -
 packages/react-devtools-inline/playwright.config.js        | 3 ++-
 scripts/circleci/run_devtools_e2e_tests.js                 | 7 +++++--
 4 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 591cf4adece0..6fa0208d682e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -291,6 +291,8 @@ jobs:
       - run:
           name: Cleanup build regression folder
           command: rm -r ./build-regression
+      - store_artifacts:
+          path: ./tmp/screenshots
 
   yarn_lint_build:
     docker: *docker
diff --git a/packages/react-devtools-inline/__tests__/__e2e__/components.test.js b/packages/react-devtools-inline/__tests__/__e2e__/components.test.js
index aaa201305b93..4df0318b0fce 100644
--- a/packages/react-devtools-inline/__tests__/__e2e__/components.test.js
+++ b/packages/react-devtools-inline/__tests__/__e2e__/components.test.js
@@ -210,7 +210,6 @@ test.describe('Components', () => {
     let count = await getComponentSearchResultsCount();
     expect(count).toBe('1 | 4');
 
-    await focusComponentSearch();
     page.keyboard.insertText('Item');
     count = await getComponentSearchResultsCount();
     expect(count).toBe('1 | 3');
diff --git a/packages/react-devtools-inline/playwright.config.js b/packages/react-devtools-inline/playwright.config.js
index 5e0f333bf29a..9054af7ccf5a 100644
--- a/packages/react-devtools-inline/playwright.config.js
+++ b/packages/react-devtools-inline/playwright.config.js
@@ -18,9 +18,10 @@ const config = {
     react_version: process.env.REACT_VERSION
       ? semver.coerce(process.env.REACT_VERSION).version
       : reactVersion,
+    trace: 'retain-on-failure',
   },
   // Some of our e2e tests can be flaky. Retry tests to make sure the error isn't transient
-  retries: 2,
+  retries: 3,
 };
 
 module.exports = config;
diff --git a/scripts/circleci/run_devtools_e2e_tests.js b/scripts/circleci/run_devtools_e2e_tests.js
index 023c0ffc5407..3ae98c42c54f 100755
--- a/scripts/circleci/run_devtools_e2e_tests.js
+++ b/scripts/circleci/run_devtools_e2e_tests.js
@@ -9,6 +9,7 @@ const ROOT_PATH = join(__dirname, '..', '..');
 const reactVersion = process.argv[2];
 const inlinePackagePath = join(ROOT_PATH, 'packages', 'react-devtools-inline');
 const shellPackagePath = join(ROOT_PATH, 'packages', 'react-devtools-shell');
+const screenshotPath = join(ROOT_PATH, 'tmp', 'screenshots');
 
 let buildProcess = null;
 let serverProcess = null;
@@ -115,9 +116,11 @@ function runTestShell() {
 async function runEndToEndTests() {
   logBright('Running e2e tests');
   if (!reactVersion) {
-    testProcess = spawn('yarn', ['test:e2e'], {cwd: inlinePackagePath});
+    testProcess = spawn('yarn', ['test:e2e', `--output=${screenshotPath}`], {
+      cwd: inlinePackagePath,
+    });
   } else {
-    testProcess = spawn('yarn', ['test:e2e'], {
+    testProcess = spawn('yarn', ['test:e2e', `--output=${screenshotPath}`], {
       cwd: inlinePackagePath,
       env: {...process.env, REACT_VERSION: reactVersion},
     });