feat(gatsby, gatsby-plugin-utils, gatsby-source-wordpress, gatsby-source-contentful, gatsby-source-drupal): Add setRequestHeaders action/api

docs/docs/how-to/plugins-and-themes/creating-a-source-plugin.md

@@ -1061,6 +1061,27 @@ You might notice that `width`, `height`, `resize`, and `gatsbyImage` can be null
 
 The string returned from `gatsbyImage` is intended to work seamlessly with [Gatsby Image Component](/docs/reference/built-in-components/gatsby-plugin-image/#gatsbyimage) just like `gatsbyImageData` does.
 
+#### Adding Image CDN request headers with the `setRequestHeaders` action
+
+Since Gatsby will be fetching files from your CMS instead of your source plugin fetching those files, you may need to set request headers for Gatsby to use in those requests.
+This is needed if for example your CMS is locked down behind some kind of authentication.
+For each domain Image CDN will make requests to, set the required headers following this example:
+
+```js
+exports.onPluginInit = ({ actions }, pluginOptions) => {
+  if (typeof actions.setRequestHeaders === `function`) {
+    actions.setRequestHeaders({
+      // set the domain the headers should apply to
+      domain: pluginOptions.apiUrl,
+      headers: {
+        // add any needed headers
+        Authorization: pluginOptions.authToken,
+      },
+    })
+  }
+}
+```
+
 #### `sourceNodes` node API additions
 
 When creating nodes, you must add some fields to the node itself to match what the `RemoteFile` interface expects. You will need `url`, `mimeType`, `filename` as mandatory fields. When you have an image type, `width` and `height` are required as well. The optional fields are `placeholderUrl` and `filesize`. `placeholderUrl` will be the url used to generate blurred or dominant color placeholder so it should contain `%width%` and `%height%` url params if possible.

packages/gatsby-core-utils/src/fetch-remote-file.ts

@@ -200,6 +200,7 @@ async function fetchFile({
     // See if there's response headers for this url
     // from a previous request.
     const headers = { ...httpHeaders }
+
     if (cachedEntry?.headers?.etag && (await fs.pathExists(filename))) {
       // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
       headers[`If-None-Match`] = cachedEntry.headers.etag

packages/gatsby-plugin-utils/src/polyfill-remote-file/__tests__/gatsby-image-resolver.ts

@@ -637,6 +637,54 @@ describe(`gatsbyImageData`, () => {
     expect(parsedFullWidthSrcSet[1].descriptor).toEqual(`700w`)
   })
 
+  it(`should fetch placeholder file with headers from the setRequestHeaders action`, async () => {
+    fetchRemoteFile.mockImplementationOnce(input => {
+      if (
+        !input.httpHeaders ||
+        input.httpHeaders.Authorization !== `Bearer 123`
+      ) {
+        throw Error(`No headers found for url ${input.url}`)
+      } else {
+        return path.join(__dirname, `__fixtures__`, `dog-portrait.jpg`)
+      }
+    })
+
+    const { store } = jest.requireActual(`gatsby/dist/redux`)
+    const { actions } = jest.requireActual(`gatsby/dist/redux/actions/public`)
+
+    store.dispatch(
+      actions.setRequestHeaders(
+        {
+          domain: portraitSource.url,
+          headers: {
+            Authorization: `Bearer 123`,
+          },
+        },
+        {
+          name: `gatsby-source-test`,
+        }
+      )
+    )
+
+    const fixedResult = await gatsbyImageResolver(
+      portraitSource,
+      {
+        layout: `fixed`,
+        width: 300,
+        placeholder: PlaceholderType.BLURRED,
+      },
+      actions
+    )
+
+    expect(fetchRemoteFile).toHaveBeenCalledTimes(1)
+    expect(fetchRemoteFile).toHaveBeenCalledWith(
+      expect.objectContaining({
+        url: portraitSource.url,
+      })
+    )
+    expect(fixedResult?.placeholder).toBeTruthy()
+  })
+
   it(`should generate dominant color placeholder by default`, async () => {
     fetchRemoteFile.mockResolvedValueOnce(
       path.join(__dirname, `__fixtures__`, `dog-portrait.jpg`)

packages/gatsby-plugin-utils/src/polyfill-remote-file/http-routes.ts

@@ -5,6 +5,7 @@ import { hasFeature } from "../has-feature"
 import { ImageCDNUrlKeys } from "./utils/url-generator"
 import { getFileExtensionFromMimeType } from "./utils/mime-type-helpers"
 import { transformImage } from "./transform-images"
+import { getRequestHeadersForUrl } from "./utils/get-request-headers-for-url"
 
 import type { Application } from "express"
 
@@ -25,11 +26,15 @@ export function addImageRoutes(app: Application): Application {
       `file`
     )
 
+    const url = req.query[ImageCDNUrlKeys.URL] as string
+
     const filePath = await fetchRemoteFile({
       directory: outputDir,
-      url: req.query[ImageCDNUrlKeys.URL] as string,
+      url,
       name: req.params.filename,
+      httpHeaders: getRequestHeadersForUrl(url),
     })
+
     fs.createReadStream(filePath).pipe(res)
   })
 

packages/gatsby-plugin-utils/src/polyfill-remote-file/jobs/gatsby-worker.ts

@@ -3,6 +3,7 @@ import { fetchRemoteFile } from "gatsby-core-utils/fetch-remote-file"
 import { cpuCoreCount } from "gatsby-core-utils/cpu-core-count"
 import Queue from "fastq"
 import { transformImage } from "../transform-images"
+import { getRequestHeadersForUrl } from "../utils/get-request-headers-for-url"
 
 interface IImageServiceProps {
   outputDir: Parameters<typeof transformImage>[0]["outputDir"]
@@ -35,11 +36,12 @@ export async function FILE_CDN({
 
   await fetchRemoteFile({
     directory: outputDir,
-    url: url,
+    url,
     name: path.basename(filename, ext),
     ext,
     cacheKey: contentDigest,
     excludeDigest: true,
+    httpHeaders: getRequestHeadersForUrl(url),
   })
 }
 

packages/gatsby-plugin-utils/src/polyfill-remote-file/placeholder-handler.ts

@@ -7,6 +7,7 @@ import getSharpInstance from "gatsby-sharp"
 import { getCache } from "./utils/cache"
 import { getImageFormatFromMimeType } from "./utils/mime-type-helpers"
 import type { IRemoteImageNode } from "./types"
+import { getRequestHeadersForUrl } from "./utils/get-request-headers-for-url"
 
 export enum PlaceholderType {
   BLURRED = `blurred`,
@@ -57,6 +58,7 @@ const queue = Queue<
     url,
     cacheKey: contentDigest,
     directory: tmpDir,
+    httpHeaders: getRequestHeadersForUrl(url),
   })
 
   switch (type) {

packages/gatsby-plugin-utils/src/polyfill-remote-file/transform-images.ts

@@ -4,6 +4,7 @@ import { fetchRemoteFile } from "gatsby-core-utils/fetch-remote-file"
 import { createContentDigest } from "gatsby-core-utils/create-content-digest"
 import getSharpInstance from "gatsby-sharp"
 import { getCache } from "./utils/cache"
+import { getRequestHeadersForUrl } from "./utils/get-request-headers-for-url"
 
 export interface IResizeArgs {
   width: number
@@ -48,10 +49,11 @@ export async function transformImage({
   const basename = path.basename(filename, ext)
   const filePath = await fetchRemoteFile({
     directory: cache.directory,
-    url: url,
+    url,
     name: basename,
     ext,
     cacheKey: contentDigest,
+    httpHeaders: getRequestHeadersForUrl(url),
   })
 
   const outputPath = path.join(outputDir, filename)

packages/gatsby-plugin-utils/src/polyfill-remote-file/utils/get-request-headers-for-url.ts

@@ -0,0 +1,27 @@
+import url from "url"
+import path from "path"
+import importFrom from "import-from"
+import resolveFrom from "resolve-from"
+
+import type { Headers } from "got"
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function getStore(): any {
+  const gatsbyPkgRoot = path.dirname(
+    resolveFrom(process.cwd(), `gatsby/package.json`)
+  )
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const { store } = importFrom(gatsbyPkgRoot, `gatsby/dist/redux`) as any
+
+  return store
+}
+
+export function getRequestHeadersForUrl(
+  passedUrl: string
+): Headers | undefined {
+  const baseDomain = url.parse(passedUrl).hostname
+  const { requestHeaders } = getStore().getState()
+
+  return baseDomain ? requestHeaders?.get(baseDomain) : undefined
+}

packages/gatsby-source-wordpress/src/gatsby-node.ts

@@ -7,6 +7,7 @@ module.exports = runApisInSteps({
     steps.setGatsbyApiToState,
     steps.setErrorMap,
     steps.tempPreventMultipleInstances,
+    steps.setRequestHeaders,
   ],
 
   pluginOptionsSchema: steps.pluginOptionsSchema,

packages/gatsby-source-wordpress/src/steps/index.ts

@@ -20,3 +20,5 @@ export {
 export { pluginOptionsSchema } from "~/steps/declare-plugin-options-schema"
 export { logPostBuildWarnings } from "~/steps/log-post-build-warnings"
 export { imageRoutes } from "~/steps/image-routes"
+
+export { setRequestHeaders } from "./set-request-headers"

packages/gatsby-source-wordpress/src/steps/set-request-headers.ts

@@ -0,0 +1,25 @@
+import btoa from "btoa"
+
+import { getPluginOptions } from "~/utils/get-gatsby-api"
+
+import type { Step } from "~/utils/run-steps"
+
+export const setRequestHeaders: Step = ({ actions }): void => {
+  if (typeof actions?.setRequestHeaders !== `function`) {
+    return
+  }
+
+  const pluginOptions = getPluginOptions()
+
+  const { auth, url } = pluginOptions
+  const { password, username } = auth?.htaccess || {}
+
+  if (password && username) {
+    actions.setRequestHeaders({
+      domain: url,
+      headers: {
+        Authorization: `Basic ${btoa(`${username}:${password}`)}`,
+      },
+    })
+  }
+}

packages/gatsby/src/redux/__tests__/__snapshots__/set-request-headers.ts.snap

@@ -0,0 +1,21 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Add request headers allows you to add request headers 1`] = `
+Object {
+  "payload": Object {
+    "domain": "testdomain.com",
+    "headers": Object {
+      "X-Header": "test",
+    },
+  },
+  "type": "SET_REQUEST_HEADERS",
+}
+`;
+
+exports[`Add request headers allows you to add request headers 2`] = `
+Map {
+  "testdomain.com" => Object {
+    "X-Header": "test",
+  },
+}
+`;

packages/gatsby/src/redux/__tests__/set-request-headers.ts

@@ -0,0 +1,68 @@
+import { setRequestHeadersReducer as reducer } from "../reducers/set-request-headers"
+import { actions } from "../actions"
+import * as reporter from "gatsby-cli/lib/reporter"
+
+jest.mock(`gatsby-cli/lib/reporter`, () => {
+  return {
+    __esModule: true,
+    warn: jest.fn(),
+  }
+})
+
+const testPlugin = {
+  name: `gatsby-source-test`,
+}
+
+describe(`Add request headers`, () => {
+  it(`allows you to add request headers`, () => {
+    const action = actions.setRequestHeaders(
+      {
+        domain: `https://testdomain.com/subpath`,
+        headers: {
+          "X-Header": `test`,
+        },
+      },
+      testPlugin
+    )
+    expect(action.payload.domain).toBe(`testdomain.com`)
+    expect(action.payload.headers).toEqual({
+      "X-Header": `test`,
+    })
+    expect(action).toMatchSnapshot()
+
+    const state = reducer(undefined, action)
+
+    expect(state.get(`testdomain.com`)).toEqual({
+      "X-Header": `test`,
+    })
+    expect(state).toMatchSnapshot()
+  })
+
+  it(`fails if domain is missing`, () => {
+    actions.setRequestHeaders(
+      {
+        headers: {
+          "X-Header": `test`,
+        },
+      },
+      testPlugin
+    )
+
+    expect(reporter.warn).toHaveBeenCalledWith(
+      `Plugin gatsby-source-test called actions.setRequestHeaders with a domain property that isn't a string.`
+    )
+  })
+
+  it(`fails if headers are missing`, () => {
+    actions.setRequestHeaders(
+      {
+        domain: `https://testdomain.com/subpath`,
+      },
+      testPlugin
+    )
+
+    expect(reporter.warn).toHaveBeenCalledWith(
+      `Plugin gatsby-source-test called actions.setRequestHeaders with a headers property that isn't an object.`
+    )
+  })
+})

packages/gatsby/src/redux/actions/public.js

@@ -1,4 +1,5 @@
 // @flow
+const reporter = require(`gatsby-cli/lib/reporter`)
 const chalk = require(`chalk`)
 const _ = require(`lodash`)
 const { stripIndent } = require(`common-tags`)
@@ -1478,4 +1479,50 @@ actions.unstable_createNodeManifest = (
   }
 }
 
+/**
+ * Stores request headers for a given domain to be later used when making requests for Image CDN (and potentially other features).
+ *
+ * @param {Object} $0
+ * @param {string} $0.domain The domain to store the headers for.
+ * @param {Object} $0.headers The headers to store.
+ */
+actions.setRequestHeaders = ({ domain, headers }, plugin: Plugin) => {
+  const noHeaders = typeof headers !== `object`
+  const noDomain = typeof domain !== `string`
+
+  if (noHeaders) {
+    reporter.warn(
+      `Plugin ${plugin.name} called actions.setRequestHeaders with a headers property that isn't an object.`
+    )
+  }
+
+  if (noDomain) {
+    reporter.warn(
+      `Plugin ${plugin.name} called actions.setRequestHeaders with a domain property that isn't a string.`
+    )
+  }
+
+  if (noDomain || noHeaders) {
+    return null
+  }
+
+  const baseDomain = url.parse(domain)?.hostname
+
+  if (baseDomain) {
+    return {
+      type: `SET_REQUEST_HEADERS`,
+      payload: {
+        domain: baseDomain,
+        headers,
+      },
+    }
+  } else {
+    reporter.warn(
+      `Plugin ${plugin.name} called actions.setRequestHeaders with a domain that is not a valid URL. (${domain})`
+    )
+
+    return null
+  }
+}
+
 module.exports = { actions }