feat(gatsby, gatsby-plugin-utils, gatsby-source-wordpress, gatsby-sou…

…rce-contentful, gatsby-source-drupal): Add setRequestHeaders action/api (#35655)

* start adding new action

* Add request headers api

* types are in src

* Add source plugin docs

* try removing type imports

* any

* move validation from reducer into setRequestHeaders action

* add comments

* move headers into each place fetchRemoteFile is called instead of inside fetchRemoteFile

* for cases where gatsby core doesn't have this yet

* add tests

* use import from to get the gatsby store

* try any

* Add action type to action union

* Update index.d.ts

* move test to the bottom so the state doesn't effect the other tests

* move getRequestHeadersForUrl into gatsby/utils

* it seems some test generates these

* remove snapshots and expect size of request headers Map

* Revert "move getRequestHeadersForUrl into gatsby/utils"

This reverts commit 4ba7177.

* Update get-request-headers-for-url.ts

* review changes

* pass store down instead of importing it

* remove console log

* add missing actions

* add store to addRemoteFilePolyfillInterface

* add httpHeaders to FILE_CDN job

* unused import

* more missing stores and pass httpHeaders into job & transformImage instead of store

* remove reporter import

* Update get-request-headers-for-url.ts

* fix tests

* fix more tests

* update warning message

* fix store type as it may not exist

* pass the actual request url, not the url url param

* Update placeholder-handler.ts

* show test paths on output

* debugging

* debugging

* add missing btoa fn

* don't use btoa, use a buffer instead

* group imports

* remove store & actions requires

Co-authored-by: Ward Peeters <ward@coding-tech.com>

docs/docs/how-to/plugins-and-themes/creating-a-source-plugin.md

@@ -1027,7 +1027,7 @@ It is also recommended that you add a polyfill to provide support back through G
 ```js
 import { addRemoteFilePolyfillInterface } from "gatsby-plugin-utils/polyfill-remote-file"
 
-exports.createSchemaCustomization = ({ actions, schema }) => {
+exports.createSchemaCustomization = ({ actions, schema, store }) => {
   const imageAssetType = addRemoteFilePolyfillInterface(
     schema.buildObjectType({
       name: `YourImageAssetNodeType`,
@@ -1039,6 +1039,7 @@ exports.createSchemaCustomization = ({ actions, schema }) => {
     {
       schema,
       actions,
+      store,
     }
   )
 
@@ -1061,6 +1062,27 @@ You might notice that `width`, `height`, `resize`, and `gatsbyImage` can be null
 
 The string returned from `gatsbyImage` is intended to work seamlessly with [Gatsby Image Component](/docs/reference/built-in-components/gatsby-plugin-image/#gatsbyimage) just like `gatsbyImageData` does.
 
+#### Adding Image CDN request headers with the `setRequestHeaders` action
+
+Since Gatsby will be fetching files from your CMS instead of your source plugin fetching those files, you may need to set request headers for Gatsby to use in those requests.
+This is needed if for example your CMS is locked down behind some kind of authentication.
+For each domain Image CDN will make requests to, set the required headers following this example:
+
+```js
+exports.onPluginInit = ({ actions }, pluginOptions) => {
+  if (typeof actions.setRequestHeaders === `function`) {
+    actions.setRequestHeaders({
+      // set the domain the headers should apply to
+      domain: pluginOptions.apiUrl,
+      headers: {
+        // add any needed headers
+        Authorization: pluginOptions.authToken,
+      },
+    })
+  }
+}
+```
+
 #### `sourceNodes` node API additions
 
 When creating nodes, you must add some fields to the node itself to match what the `RemoteFile` interface expects. You will need `url`, `mimeType`, `filename` as mandatory fields. When you have an image type, `width` and `height` are required as well. The optional fields are `placeholderUrl` and `filesize`. `placeholderUrl` will be the url used to generate blurred or dominant color placeholder so it should contain `%width%` and `%height%` url params if possible.
@@ -1087,8 +1109,8 @@ Add the polyfill, `polyfillImageServiceDevRoutes`, to ensure that the developmen
 ```js
 import { polyfillImageServiceDevRoutes } from "gatsby-plugin-utils/polyfill-remote-file"
 
-export const onCreateDevServer = ({ app }) => {
-  polyfillImageServiceDevRoutes(app)
+export const onCreateDevServer = ({ app, store }) => {
+  polyfillImageServiceDevRoutes(app, store)
 }
 ```
 

e2e-tests/development-runtime/gatsby-node.js

@@ -17,6 +17,7 @@ exports.createSchemaCustomization = ({ actions, schema, store }) => {
       {
         store,
         schema,
+        actions,
       }
     )
   )
@@ -27,8 +28,7 @@ exports.sourceNodes = ({ actions, createNodeId, createContentDigest }) => {
   const items = [
     {
       name: "photoA.jpg",
-      url:
-        "https://images.unsplash.com/photo-1517849845537-4d257902454a?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2000&q=80",
+      url: "https://images.unsplash.com/photo-1517849845537-4d257902454a?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2000&q=80",
       placeholderUrl:
         "https://images.unsplash.com/photo-1517849845537-4d257902454a?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=%width%&h=%height%",
       mimeType: "image/jpg",
@@ -38,17 +38,15 @@ exports.sourceNodes = ({ actions, createNodeId, createContentDigest }) => {
     },
     {
       name: "photoB.jpg",
-      url:
-        "https://images.unsplash.com/photo-1552053831-71594a27632d?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&h=2000&q=10",
+      url: "https://images.unsplash.com/photo-1552053831-71594a27632d?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&h=2000&q=10",
       mimeType: "image/jpg",
       filename: "photo-1552053831.jpg",
       width: 1247,
       height: 2000,
     },
     {
       name: "photoC.jpg",
-      url:
-        "https://images.unsplash.com/photo-1561037404-61cd46aa615b?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2000&q=80",
+      url: "https://images.unsplash.com/photo-1561037404-61cd46aa615b?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2000&q=80",
       placeholderUrl:
         "https://images.unsplash.com/photo-1561037404-61cd46aa615b?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=%width%&h=%height%",
       mimeType: "image/jpg",
@@ -275,6 +273,6 @@ exports.createResolvers = ({ createResolvers }) => {
 }
 
 /** @type{import('gatsby').onCreateDevServer} */
-exports.onCreateDevServer = ({ app }) => {
-  polyfillImageServiceDevRoutes(app)
+exports.onCreateDevServer = ({ app, store }) => {
+  polyfillImageServiceDevRoutes(app, store)
 }

jest.config.js

@@ -56,7 +56,7 @@ module.exports = {
   snapshotSerializers: [`jest-serializer-path`],
   collectCoverageFrom: coverageDirs,
   reporters: process.env.CI
-    ? [[`jest-silent-reporter`, { useDots: true }]].concat(
+    ? [[`jest-silent-reporter`, { useDots: true, showPaths: true }]].concat(
         useCoverage ? `jest-junit` : []
       )
     : [`default`].concat(useCoverage ? `jest-junit` : []),

packages/gatsby-core-utils/src/fetch-remote-file.ts

@@ -200,6 +200,7 @@ async function fetchFile({
     // See if there's response headers for this url
     // from a previous request.
     const headers = { ...httpHeaders }
+
     if (cachedEntry?.headers?.etag && (await fs.pathExists(filename))) {
       // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
       headers[`If-None-Match`] = cachedEntry.headers.etag

packages/gatsby-plugin-utils/README.md

@@ -95,7 +95,7 @@ const {
   polyfillImageServiceDevRoutes,
 } = require(`gatsby-plugin-utils/pollyfill-remote-file`)
 
-exports.createSchemaCustomization ({ actions, schema }) => {
+exports.createSchemaCustomization ({ actions, schema, store }) => {
   actions.createTypes([
     addRemoteFilePolyfillInterface(
       schema.buildObjectType({
@@ -108,13 +108,14 @@ exports.createSchemaCustomization ({ actions, schema }) => {
       {
         schema,
         actions,
+        store
       }
     )
   ]);
 }
 
 /** @type {import('gatsby').onCreateDevServer} */
-exports.onCreateDevServer = ({ app }) => {
-  polyfillImageServiceDevRoutes(app)
+exports.onCreateDevServer = ({ app, store }) => {
+  polyfillImageServiceDevRoutes(app, store)
 }
 ```

packages/gatsby-plugin-utils/src/polyfill-remote-file/__tests__/gatsby-image-resolver.ts

@@ -1,12 +1,13 @@
 import path from "path"
+import url from "url"
 import { ensureDir, remove } from "fs-extra"
 import importFrom from "import-from"
 import { fetchRemoteFile } from "gatsby-core-utils/fetch-remote-file"
 import { gatsbyImageResolver } from "../index"
 import * as dispatchers from "../jobs/dispatchers"
 import { PlaceholderType } from "../placeholder-handler"
 import { generateImageUrl } from "../utils/url-generator"
-import type { Actions } from "gatsby"
+import type { Actions, Store } from "gatsby"
 
 jest.spyOn(dispatchers, `shouldDispatch`).mockImplementation(() => false)
 jest.mock(`import-from`)
@@ -36,6 +37,14 @@ function parseSrcSet(
   })
 }
 
+const store = {
+  getState: (): { requestHeaders: Map<string, Record<string, string>> } => {
+    return {
+      requestHeaders: new Map(),
+    }
+  },
+} as unknown as Store
+
 describe(`gatsbyImageData`, () => {
   const cacheDir = path.join(__dirname, `.cache`)
 
@@ -86,7 +95,8 @@ describe(`gatsbyImageData`, () => {
         width: 300,
         placeholder: `none`,
       },
-      actions
+      actions,
+      store
     )
 
     const parsedSrcSet = parseSrcSet(result.images.sources[0].srcSet)
@@ -120,7 +130,8 @@ describe(`gatsbyImageData`, () => {
         placeholder: `none`,
         cropFocus: [`entropy`],
       },
-      actions
+      actions,
+      store
     )
     const parsedSrcSet = parseSrcSet(result.images.sources[0].srcSet)
     expect(parsedSrcSet[0].src).toEqual(
@@ -159,7 +170,8 @@ describe(`gatsbyImageData`, () => {
         },
         // @ts-ignore - don't care
         {},
-        actions
+        actions,
+        store
       )
     ).toBe(null)
     expect(dispatchers.shouldDispatch).not.toHaveBeenCalled()
@@ -173,7 +185,8 @@ describe(`gatsbyImageData`, () => {
         width: 300,
         placeholder: `none`,
       },
-      actions
+      actions,
+      store
     )
 
     const parsedSrcSet = parseSrcSet(result.images.sources[0].srcSet)
@@ -247,7 +260,8 @@ describe(`gatsbyImageData`, () => {
         width: 300,
         placeholder: `none`,
       },
-      actions
+      actions,
+      store
     )
 
     const parsedSrcSet = parseSrcSet(result.images.sources[0].srcSet)
@@ -357,7 +371,8 @@ describe(`gatsbyImageData`, () => {
         width: 2000,
         placeholder: `none`,
       },
-      actions
+      actions,
+      store
     )
 
     const parsedSrcSet = parseSrcSet(result.images.sources[0].srcSet)
@@ -464,7 +479,8 @@ describe(`gatsbyImageData`, () => {
         placeholder: `none`,
         outputPixelDensities: [1, 2],
       },
-      actions
+      actions,
+      store
     )
     const constrainedResult = await gatsbyImageResolver(
       portraitSource,
@@ -474,7 +490,8 @@ describe(`gatsbyImageData`, () => {
         placeholder: `none`,
         outputPixelDensities: [1, 2],
       },
-      actions
+      actions,
+      store
     )
     const fullWidthResult = await gatsbyImageResolver(
       {
@@ -488,7 +505,8 @@ describe(`gatsbyImageData`, () => {
         placeholder: `none`,
         outputPixelDensities: [1, 2],
       },
-      actions
+      actions,
+      store
     )
 
     const parsedFixedSrcSet = parseSrcSet(fixedResult.images.sources[0].srcSet)
@@ -576,7 +594,8 @@ describe(`gatsbyImageData`, () => {
         layout: `constrained`,
         placeholder: `none`,
       },
-      actions
+      actions,
+      store
     )
 
     expect(result.images.fallback.src).not.toContain(` `)
@@ -597,7 +616,8 @@ describe(`gatsbyImageData`, () => {
         placeholder: `none`,
         breakpoints: [350, 700],
       },
-      actions
+      actions,
+      store
     )
     const constrainedResult = await gatsbyImageResolver(
       biggerPortraitSource,
@@ -607,7 +627,8 @@ describe(`gatsbyImageData`, () => {
         placeholder: `none`,
         breakpoints: [350, 700],
       },
-      actions
+      actions,
+      store
     )
     const fullWidthResult = await gatsbyImageResolver(
       biggerPortraitSource,
@@ -617,7 +638,8 @@ describe(`gatsbyImageData`, () => {
         placeholder: `none`,
         breakpoints: [350, 700],
       },
-      actions
+      actions,
+      store
     )
 
     const parsedFixedSrcSet = parseSrcSet(fixedResult.images.sources[0].srcSet)
@@ -647,7 +669,8 @@ describe(`gatsbyImageData`, () => {
         layout: `fixed`,
         width: 300,
       },
-      actions
+      actions,
+      store
     )
 
     expect(fetchRemoteFile).toHaveBeenCalledTimes(1)
@@ -665,7 +688,8 @@ describe(`gatsbyImageData`, () => {
         width: 300,
         placeholder: PlaceholderType.BLURRED,
       },
-      actions
+      actions,
+      store
     )
 
     expect(fetchRemoteFile).toHaveBeenCalledTimes(1)
@@ -692,7 +716,8 @@ describe(`gatsbyImageData`, () => {
         width: 300,
         placeholder: PlaceholderType.TRACED_SVG,
       },
-      actions
+      actions,
+      store
     )
 
     expect(fetchRemoteFile).toHaveBeenCalledTimes(1)
@@ -717,7 +742,8 @@ describe(`gatsbyImageData`, () => {
         placeholder: `none`,
         outputPixelDensities: [1, 2],
       },
-      actions
+      actions,
+      store
     )
 
     expect(constrainedResult?.images.sources[0].type).toBe(`image/avif`)
@@ -726,4 +752,45 @@ describe(`gatsbyImageData`, () => {
 
     expect(constrainedResult?.images.sources.length).toBe(2)
   })
+
+  it(`should fetch placeholder file with headers from the setRequestHeaders action`, async () => {
+    const authToken = `Bearer 12345`
+
+    fetchRemoteFile.mockImplementationOnce(input => {
+      if (!input.httpHeaders || input.httpHeaders.Authorization !== authToken) {
+        throw Error(`No headers found for url ${input.url}`)
+      } else {
+        return path.join(__dirname, `__fixtures__`, `dog-portrait.jpg`)
+      }
+    })
+
+    const baseDomain = url.parse(portraitSource.url)?.hostname
+
+    const store = {
+      getState: (): { requestHeaders: Map<string, Record<string, string>> } => {
+        return {
+          requestHeaders: new Map([[baseDomain, { Authorization: authToken }]]),
+        }
+      },
+    } as unknown as Store
+
+    const fixedResult = await gatsbyImageResolver(
+      portraitSource,
+      {
+        layout: `fixed`,
+        width: 300,
+        placeholder: PlaceholderType.BLURRED,
+      },
+      actions,
+      store
+    )
+
+    expect(fetchRemoteFile).toHaveBeenCalledTimes(1)
+    expect(fetchRemoteFile).toHaveBeenCalledWith(
+      expect.objectContaining({
+        url: portraitSource.url,
+      })
+    )
+    expect(fixedResult?.placeholder).toBeTruthy()
+  })
 })