Update redbox tests (#20866)

This adds retrying for getting redbox content and ensures the `next.config.js` file is cleaned up for the dynamic-routing test suite.

x-ref: #20786 (comment)

test/integration/client-navigation/test/rendering.js

@@ -245,7 +245,7 @@ export default function (render, fetch, ctx) {
       const expectedErrorMessage =
         'Circular structure in "getInitialProps" result of page "/circular-json-error".'
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       const text = await getRedboxHeader(browser)
       expect(text).toContain(expectedErrorMessage)
     })
@@ -259,7 +259,7 @@ export default function (render, fetch, ctx) {
       const expectedErrorMessage =
         '"InstanceInitialPropsPage.getInitialProps()" is defined as an instance method - visit https://err.sh/vercel/next.js/get-initial-props-as-an-instance-method for more information.'
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       const text = await getRedboxHeader(browser)
       expect(text).toContain(expectedErrorMessage)
     })
@@ -269,7 +269,7 @@ export default function (render, fetch, ctx) {
       const expectedErrorMessage =
         '"EmptyInitialPropsPage.getInitialProps()" should resolve to an object. But found "null" instead.'
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       const text = await getRedboxHeader(browser)
       expect(text).toContain(expectedErrorMessage)
     })
@@ -305,22 +305,22 @@ export default function (render, fetch, ctx) {
 
     test('default export is not a React Component', async () => {
       const browser = await webdriver(ctx.appPort, '/no-default-export')
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       const text = await getRedboxHeader(browser)
       expect(text).toMatch(/The default export is not a React Component/)
     })
 
     test('error-inside-page', async () => {
       const browser = await webdriver(ctx.appPort, '/error-inside-page')
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       const text = await getRedboxHeader(browser)
       expect(text).toMatch(/This is an expected error/)
       // Sourcemaps are applied by react-error-overlay, so we can't check them on SSR.
     })
 
     test('error-in-the-global-scope', async () => {
       const browser = await webdriver(ctx.appPort, '/error-in-the-global-scope')
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       const text = await getRedboxHeader(browser)
       expect(text).toMatch(/aa is not defined/)
       // Sourcemaps are applied by react-error-overlay, so we can't check them on SSR.
@@ -433,7 +433,7 @@ export default function (render, fetch, ctx) {
 
     it('should show a valid error when undefined is thrown', async () => {
       const browser = await webdriver(ctx.appPort, '/throw-undefined')
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       const text = await getRedboxHeader(browser)
 
       expect(text).toContain(

test/integration/dynamic-routing/test/index.test.js

@@ -789,7 +789,7 @@ function runTests(dev) {
       await browser
         .elementByCss('#view-post-1-interpolated-incorrectly')
         .click()
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       const header = await getRedboxHeader(browser)
       expect(header).toContain(
         'The provided `href` (/[name]?another=value) value is missing query values (name) to be interpolated properly.'
@@ -1096,6 +1096,7 @@ describe('Dynamic Routing', () => {
     })
     afterAll(async () => {
       await killApp(app)
+      await fs.remove(nextConfig)
     })
     runTests()
   })

test/integration/image-component/base-path/test/index.test.js

@@ -405,7 +405,7 @@ function runTests(mode) {
     it('should show missing src error', async () => {
       const browser = await webdriver(appPort, '/docs/missing-src')
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       expect(await getRedboxHeader(browser)).toContain(
         'Image is missing required "src" property. Make sure you pass "src" in props to the `next/image` component. Received: {"width":200}'
       )
@@ -414,7 +414,7 @@ function runTests(mode) {
     it('should show invalid src error', async () => {
       const browser = await webdriver(appPort, '/docs/invalid-src')
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       expect(await getRedboxHeader(browser)).toContain(
         'Invalid src prop (https://google.com/test.png) on `next/image`, hostname "google.com" is not configured under images in your `next.config.js`'
       )
@@ -426,7 +426,7 @@ function runTests(mode) {
         '/docs/invalid-src-proto-relative'
       )
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       expect(await getRedboxHeader(browser)).toContain(
         'Failed to parse src "//assets.example.com/img.jpg" on `next/image`, protocol-relative URL (//) must be changed to an absolute URL (http:// or https://)'
       )
@@ -435,7 +435,7 @@ function runTests(mode) {
     it('should show invalid unsized error', async () => {
       const browser = await webdriver(appPort, '/docs/invalid-unsized')
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       expect(await getRedboxHeader(browser)).toContain(
         'Image with src "/docs/test.png" has deprecated "unsized" property, which was removed in favor of the "layout=\'fill\'" property'
       )

test/integration/image-component/default/test/index.test.js

@@ -463,7 +463,7 @@ function runTests(mode) {
     it('should show missing src error', async () => {
       const browser = await webdriver(appPort, '/missing-src')
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       expect(await getRedboxHeader(browser)).toContain(
         'Image is missing required "src" property. Make sure you pass "src" in props to the `next/image` component. Received: {"width":200}'
       )
@@ -472,7 +472,7 @@ function runTests(mode) {
     it('should show invalid src error', async () => {
       const browser = await webdriver(appPort, '/invalid-src')
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       expect(await getRedboxHeader(browser)).toContain(
         'Invalid src prop (https://google.com/test.png) on `next/image`, hostname "google.com" is not configured under images in your `next.config.js`'
       )
@@ -481,7 +481,7 @@ function runTests(mode) {
     it('should show invalid src error when protocol-relative', async () => {
       const browser = await webdriver(appPort, '/invalid-src-proto-relative')
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       expect(await getRedboxHeader(browser)).toContain(
         'Failed to parse src "//assets.example.com/img.jpg" on `next/image`, protocol-relative URL (//) must be changed to an absolute URL (http:// or https://)'
       )
@@ -490,7 +490,7 @@ function runTests(mode) {
     it('should show invalid unsized error', async () => {
       const browser = await webdriver(appPort, '/invalid-unsized')
 
-      await hasRedbox(browser)
+      expect(await hasRedbox(browser)).toBe(true)
       expect(await getRedboxHeader(browser)).toContain(
         'Image with src "/test.png" has deprecated "unsized" property, which was removed in favor of the "layout=\'fill\'" property'
       )

test/lib/next-test-utils.js

@@ -444,6 +444,33 @@ export async function evaluate(browser, input) {
   }
 }
 
+export async function retry(fn, duration = 3000, interval = 500, description) {
+  if (duration % interval !== 0) {
+    throw new Error(
+      `invalid duration ${duration} and interval ${interval} mix, duration must be evenly divisible by interval`
+    )
+  }
+
+  for (let i = duration; i >= 0; i -= interval) {
+    try {
+      return await fn()
+    } catch (err) {
+      if (i === 0) {
+        console.error(
+          `Failed to retry${
+            description ? ` ${description}` : ''
+          } within ${duration}ms`
+        )
+        throw err
+      }
+      console.warn(
+        `Retrying${description ? ` ${description}` : ''} in ${interval}ms`
+      )
+      await waitFor(interval)
+    }
+  }
+}
+
 export async function hasRedbox(browser, expected = true) {
   let attempts = 30
   do {
@@ -471,31 +498,43 @@ export async function hasRedbox(browser, expected = true) {
 }
 
 export async function getRedboxHeader(browser) {
-  return evaluate(browser, () => {
-    const portal = [].slice
-      .call(document.querySelectorAll('nextjs-portal'))
-      .find((p) => p.shadowRoot.querySelector('[data-nextjs-dialog-header'))
-    const root = portal.shadowRoot
-    return root
-      .querySelector('[data-nextjs-dialog-header]')
-      .innerText.replace(/__WEBPACK_DEFAULT_EXPORT__/, 'Unknown')
-  })
+  return retry(
+    () =>
+      evaluate(browser, () => {
+        const portal = [].slice
+          .call(document.querySelectorAll('nextjs-portal'))
+          .find((p) => p.shadowRoot.querySelector('[data-nextjs-dialog-header'))
+        const root = portal.shadowRoot
+        return root
+          .querySelector('[data-nextjs-dialog-header]')
+          .innerText.replace(/__WEBPACK_DEFAULT_EXPORT__/, 'Unknown')
+      }),
+    3000,
+    500,
+    'getRedboxHeader'
+  )
 }
 
 export async function getRedboxSource(browser) {
-  return evaluate(browser, () => {
-    const portal = [].slice
-      .call(document.querySelectorAll('nextjs-portal'))
-      .find((p) =>
-        p.shadowRoot.querySelector(
-          '#nextjs__container_errors_label, #nextjs__container_build_error_label'
-        )
-      )
-    const root = portal.shadowRoot
-    return root
-      .querySelector('[data-nextjs-codeframe], [data-nextjs-terminal]')
-      .innerText.replace(/__WEBPACK_DEFAULT_EXPORT__/, 'Unknown')
-  })
+  return retry(
+    () =>
+      evaluate(browser, () => {
+        const portal = [].slice
+          .call(document.querySelectorAll('nextjs-portal'))
+          .find((p) =>
+            p.shadowRoot.querySelector(
+              '#nextjs__container_errors_label, #nextjs__container_build_error_label'
+            )
+          )
+        const root = portal.shadowRoot
+        return root
+          .querySelector('[data-nextjs-codeframe], [data-nextjs-terminal]')
+          .innerText.replace(/__WEBPACK_DEFAULT_EXPORT__/, 'Unknown')
+      }),
+    3000,
+    500,
+    'getRedboxSource'
+  )
 }
 
 export function getBrowserBodyText(browser) {