From 7dfd52d9b43e9fe2e456b273dc75de99634ace5d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 3 Jun 2022 09:55:56 -0700 Subject: [PATCH] feat(gatsby): Exclude the internal/fields objects for greater consistency creating contentDigest (#33671) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(gatsby-core-utils): Exclude the internal object for greater consistency I was puzzled why a site was having more nodes than expected by marked as changed on each data update. I use jest-diff to show the diff between the nodes and as it turns out, it was all trivial differences in the `internal` object. Since the `internal` object is managed by the framework and source/transformer plugins typically don't ensure that the `internal` object hasn't changed between updates — we should just exclude it when generating the digest. Making this change resulted in ~20 less nodes being changed which reduced the number of invalidated queries from ~40k to ~6k, dramatically speeding up the incremental builds. I tested whether the destructuring would slow things down any by running the ~700k nodes on this site through both the old and new version of the site. They were essentially the same — ~16.5s vs. ~16.9s with the new algorithm being faster, presumably as we don't now have to hash the internal object. I expect this will result in fewer changed nodes in a lot of plugins. * Update create-content-digest.ts * Only hash as node if node & update snapshots * Only remove autogenerated fields from internal object * Move node-specific createContentDigest code to api-runner-node * revert snapshot changes * update snapshots * Update packages/gatsby/src/utils/api-runner-node.js Co-authored-by: Michal Piechowiak Co-authored-by: Ward Peeters Co-authored-by: Michal Piechowiak --- .../source-changed/scenario.js | 8 +++--- .../transformer-added/scenario.js | 2 +- .../transformer-changed/scenario.js | 4 +-- .../transformer-removed/scenario.js | 2 +- .../source-changed/scenario.js | 4 +-- .../transformer-added/scenario.js | 2 +- .../transformer-changed/scenario.js | 2 +- packages/gatsby/src/utils/api-runner-node.js | 27 ++++++++++++++++++- 8 files changed, 38 insertions(+), 13 deletions(-) diff --git a/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/source-changed/scenario.js b/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/source-changed/scenario.js index bd729031707d2..27496bf9e313f 100644 --- a/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/source-changed/scenario.js +++ b/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/source-changed/scenario.js @@ -57,8 +57,8 @@ const nodesTest = ({ - \\"foo\\": \\"run-1\\", \\"id\\": \\"parent_parentChangeForTransformer\\", \\"internal\\": Object { - - \\"contentDigest\\": \\"9d6d458358c77dbe8f4247752ebe41f0\\", - + \\"contentDigest\\": \\"3021b9f76357d1cffb3c40fabc9e08fb\\", + - \\"contentDigest\\": \\"dc3d69faa290879f166b8cf2a459e123\\", + + \\"contentDigest\\": \\"5b2b719234b7823a8a7743409714b7a7\\", \\"owner\\": \\"source-and-transformers-child-nodes/source-changed/gatsby-source\\", \\"type\\": \\"Parent_ParentChangeForTransformer\\", }, @@ -75,8 +75,8 @@ const nodesTest = ({ + \\"foo\\": undefined, \\"id\\": \\"parent_parentChangeForTransformer >>> Child\\", \\"internal\\": Object { - - \\"contentDigest\\": \\"603e50c1fe96279688538ab046d1d70a\\", - + \\"contentDigest\\": \\"f784f9722081b56fee8ca34708299a37\\", + - \\"contentDigest\\": \\"7ec9e8b4f83ebca01990608f961b1a5d\\", + + \\"contentDigest\\": \\"4cecf7ebaf0c19b22eef8835c725c2e0\\", \\"owner\\": \\"source-and-transformers-child-nodes/source-changed/gatsby-transformer\\", \\"type\\": \\"ChildOfParent_ParentChangeForTransformer\\", }, diff --git a/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-added/scenario.js b/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-added/scenario.js index b05bfe0e41b64..9334e517a0081 100644 --- a/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-added/scenario.js +++ b/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-added/scenario.js @@ -55,7 +55,7 @@ const nodesTest = ({ \\"foo\\": \\"run-1\\", \\"id\\": \\"parent_childAdditionForTransformer\\", \\"internal\\": Object { - \\"contentDigest\\": \\"f85e860f002547e9da9e893e3e44e162\\", + \\"contentDigest\\": \\"f99c0539a6bb6061b072fe782a8e441f\\", \\"owner\\": \\"source-and-transformers-child-nodes/transformer-added/gatsby-source\\", \\"type\\": \\"Parent_ChildAdditionForTransformer\\", }, diff --git a/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-changed/scenario.js b/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-changed/scenario.js index 2d8d54196ac04..036e4c8dab5ae 100644 --- a/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-changed/scenario.js +++ b/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-changed/scenario.js @@ -78,8 +78,8 @@ const nodesTest = ({ + \\"foo\\": \\"baz\\", \\"id\\": \\"parent_childChangeForTransformer >>> Child\\", \\"internal\\": Object { - - \\"contentDigest\\": \\"25ad44d8db6f7cee17e248d3b4c94538\\", - + \\"contentDigest\\": \\"3c57fc662e03b2b443b47575f7e82ec0\\", + - \\"contentDigest\\": \\"04b03439f5e1d249d999e6763e23f306\\", + + \\"contentDigest\\": \\"66875f4e1170801ce795d9d863a12b2a\\", \\"owner\\": \\"source-and-transformers-child-nodes/transformer-changed/gatsby-transformer\\", \\"type\\": \\"ChildOfParent_ChildChangeForTransformer\\", }, diff --git a/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-removed/scenario.js b/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-removed/scenario.js index 29289874e9dd5..2f34cedcae6c4 100644 --- a/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-removed/scenario.js +++ b/integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-removed/scenario.js @@ -85,7 +85,7 @@ const nodesTest = ({ \\"foo\\": \\"run-1\\", \\"id\\": \\"parent_childDeletionForTransformer\\", \\"internal\\": Object { - \\"contentDigest\\": \\"872081fdfb66891ee6ccdcd13716a5ce\\", + \\"contentDigest\\": \\"6140f27e9454bda710ae6da2cda698f0\\", \\"owner\\": \\"source-and-transformers-child-nodes/transformer-removed/gatsby-source\\", \\"type\\": \\"Parent_ChildDeletionForTransformer\\", }, diff --git a/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/source-changed/scenario.js b/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/source-changed/scenario.js index acfbccdcf5213..805cbfdb798c2 100644 --- a/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/source-changed/scenario.js +++ b/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/source-changed/scenario.js @@ -49,8 +49,8 @@ const nodesTest = ({ - \\"foo\\": \\"run-1\\", \\"id\\": \\"parent_parentChangeForFields\\", \\"internal\\": Object { - - \\"contentDigest\\": \\"e88540d53597617cf99d612601037013\\", - + \\"contentDigest\\": \\"3b78e62e87d3f1d8e92d274aa8dbe548\\", + - \\"contentDigest\\": \\"bde8140cd815ea2bb9e7ae2b45330d08\\", + + \\"contentDigest\\": \\"76dc03ca64a18a99d1581b9e7dd224ef\\", \\"fieldOwners\\": Object { \\"bar\\": \\"source-and-transformers-node-fields/source-changed/gatsby-transformer\\", \\"foo\\": \\"source-and-transformers-node-fields/source-changed/gatsby-transformer\\", diff --git a/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-added/scenario.js b/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-added/scenario.js index 156e3b96493e1..044c98a5e3e5a 100644 --- a/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-added/scenario.js +++ b/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-added/scenario.js @@ -50,7 +50,7 @@ const nodesTest = ({ \\"foo\\": \\"run-1\\", \\"id\\": \\"parent_childAdditionForFields\\", \\"internal\\": Object { - \\"contentDigest\\": \\"bdf44fdce30b104b4f290d66c2dc3ca1\\", + \\"contentDigest\\": \\"b6d6df9ccdfc5388cde03e3140520cf5\\", + \\"fieldOwners\\": Object { + \\"foo1\\": \\"source-and-transformers-node-fields/transformer-added/gatsby-transformer\\", + }, diff --git a/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-changed/scenario.js b/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-changed/scenario.js index 681e2842c53eb..58fb36fd6c412 100644 --- a/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-changed/scenario.js +++ b/integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-changed/scenario.js @@ -67,7 +67,7 @@ const nodesTest = ({ \\"foo\\": \\"run-1\\", \\"id\\": \\"parent_childChangeForFields\\", \\"internal\\": Object { - \\"contentDigest\\": \\"fb9e9b9c26522bceaa1f51c537b2aff2\\", + \\"contentDigest\\": \\"c6c5c686a949351323ba4a04b585840a\\", \\"fieldOwners\\": Object { - \\"foo1\\": \\"source-and-transformers-node-fields/transformer-changed/gatsby-transformer\\", + \\"foo2\\": \\"source-and-transformers-node-fields/transformer-changed/gatsby-transformer\\", diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index f4226d3800b4b..3b3b7250ec2d8 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -13,7 +13,9 @@ const { codeFrameColumns } = require(`@babel/code-frame`) const fs = require(`fs-extra`) const { getCache } = require(`./get-cache`) import { createNodeId } from "./create-node-id" -const { createContentDigest } = require(`gatsby-core-utils`) +const { + createContentDigest: _createContentDigest, +} = require(`gatsby-core-utils`) import { buildObjectType, buildUnionType, @@ -32,6 +34,29 @@ const { trackBuildError, decorateEvent } = require(`gatsby-telemetry`) import errorParser from "./api-runner-error-parser" import { wrapNode, wrapNodes } from "./detect-node-mutations" +// Override createContentDigest to remove autogenerated data from nodes to +// ensure consistent digests. +function createContentDigest(node) { + if (!node?.internal?.type) { + // this doesn't look like a node, so let's just pass it as-is + return _createContentDigest(node) + } + return _createContentDigest({ + ...node, + internal: { + ...node.internal, + // Remove auto-generated fields that'd prevent + // creating a consistent contentDigest. + contentDigest: undefined, + owner: undefined, + fieldOwners: undefined, + ignoreType: undefined, + counter: undefined, + }, + fields: undefined, + }) +} + if (!process.env.BLUEBIRD_DEBUG && !process.env.BLUEBIRD_LONG_STACK_TRACES) { // Unless specified - disable longStackTraces // as this have severe perf penalty ( http://bluebirdjs.com/docs/api/promise.longstacktraces.html )