From c610e7908575c94863384fc04fd6079fb3580297 Mon Sep 17 00:00:00 2001 From: iChenLei Date: Tue, 14 Dec 2021 14:11:46 +0800 Subject: [PATCH 1/6] ci: make it crash with oom --- e2e-tests/development-runtime/content/2018-12-14-hello-world.md | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e-tests/development-runtime/content/2018-12-14-hello-world.md b/e2e-tests/development-runtime/content/2018-12-14-hello-world.md index bdedb261138b7..0ca23c6f01d33 100644 --- a/e2e-tests/development-runtime/content/2018-12-14-hello-world.md +++ b/e2e-tests/development-runtime/content/2018-12-14-hello-world.md @@ -1,6 +1,7 @@ --- title: Hello World date: 2018-12-14 +length: 81185414 # Regression for issue #26565 --- This is a truly meaningful blog post From 725725ef82055fe98fab7c6431f4a796428e349b Mon Sep 17 00:00:00 2001 From: iChenLei Date: Tue, 14 Dec 2021 14:45:29 +0800 Subject: [PATCH 2/6] fix: add fix solution --- packages/gatsby/src/utils/sanitize-node.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/gatsby/src/utils/sanitize-node.js b/packages/gatsby/src/utils/sanitize-node.js index fc33b3643caf8..661af54264024 100644 --- a/packages/gatsby/src/utils/sanitize-node.js +++ b/packages/gatsby/src/utils/sanitize-node.js @@ -14,6 +14,16 @@ const sanitizeNode = (data, isNode = true, path = new Set()) => { path.add(data) const returnData = isPlainObject ? {} : [] + // Refer to https://github.com/gatsbyjs/gatsby/issues/26565 + const hasCustomLengthFiled = isPlainObject + ? Object.prototype.hasOwnProperty.call(data, `length`) + : false + let rawLength + if (hasCustomLengthFiled) { + rawLength = data.length + delete data.length + } + let anyFieldChanged = false _.each(data, (o, key) => { if (isNode && key === `internal`) { @@ -27,6 +37,13 @@ const sanitizeNode = (data, isNode = true, path = new Set()) => { } }) + if (hasCustomLengthFiled) { + returnData[`length`] = sanitizeNode(rawLength, false, path) + if (returnData[`length`] !== rawLength) { + anyFieldChanged = true + } + } + if (anyFieldChanged) { data = omitUndefined(returnData) } From 7b1a04ea6038656c7db8d332db08f881f4e8bdf5 Mon Sep 17 00:00:00 2001 From: iChenLei Date: Tue, 14 Dec 2021 14:50:07 +0800 Subject: [PATCH 3/6] fix: recover data length field --- packages/gatsby/src/utils/sanitize-node.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/gatsby/src/utils/sanitize-node.js b/packages/gatsby/src/utils/sanitize-node.js index 661af54264024..d332b9c4d11a0 100644 --- a/packages/gatsby/src/utils/sanitize-node.js +++ b/packages/gatsby/src/utils/sanitize-node.js @@ -38,6 +38,7 @@ const sanitizeNode = (data, isNode = true, path = new Set()) => { }) if (hasCustomLengthFiled) { + data[`length`] = sanitizeNode(rawLength, false, path) returnData[`length`] = sanitizeNode(rawLength, false, path) if (returnData[`length`] !== rawLength) { anyFieldChanged = true From 8abfa428df0569e9712868299291a5ec3acadf61 Mon Sep 17 00:00:00 2001 From: LekoArts Date: Thu, 24 Nov 2022 15:17:01 +0100 Subject: [PATCH 4/6] revert test change --- e2e-tests/development-runtime/content/2018-12-14-hello-world.md | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e-tests/development-runtime/content/2018-12-14-hello-world.md b/e2e-tests/development-runtime/content/2018-12-14-hello-world.md index 0ca23c6f01d33..bdedb261138b7 100644 --- a/e2e-tests/development-runtime/content/2018-12-14-hello-world.md +++ b/e2e-tests/development-runtime/content/2018-12-14-hello-world.md @@ -1,7 +1,6 @@ --- title: Hello World date: 2018-12-14 -length: 81185414 # Regression for issue #26565 --- This is a truly meaningful blog post From 705737726269666a054571d54c9f2c8d49f7bd7b Mon Sep 17 00:00:00 2001 From: LekoArts Date: Thu, 24 Nov 2022 15:17:13 +0100 Subject: [PATCH 5/6] change function --- packages/gatsby/src/redux/actions/public.js | 2 +- packages/gatsby/src/utils/sanitize-node.ts | 71 ++++++++++++++------- 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js index 1c0d8e644655d..6e7f715b65246 100644 --- a/packages/gatsby/src/redux/actions/public.js +++ b/packages/gatsby/src/redux/actions/public.js @@ -15,7 +15,7 @@ const { const { splitComponentPath } = require(`gatsby-core-utils/parse-component-path`) const { hasNodeChanged } = require(`../../utils/nodes`) const { getNode, getDataStore } = require(`../../datastore`) -import sanitizeNode from "../../utils/sanitize-node" +import { sanitizeNode } from "../../utils/sanitize-node" const { store } = require(`../index`) const { validateComponent } = require(`../../utils/validate-component`) import { nodeSchema } from "../../joi-schemas/joi" diff --git a/packages/gatsby/src/utils/sanitize-node.ts b/packages/gatsby/src/utils/sanitize-node.ts index d86601a6a08fe..a92504a4cb0f4 100644 --- a/packages/gatsby/src/utils/sanitize-node.ts +++ b/packages/gatsby/src/utils/sanitize-node.ts @@ -3,15 +3,15 @@ import _ from "lodash" import type { IGatsbyNode } from "../redux/types" import type { GatsbyIterable } from "../datastore/common/iterable" -type data = IGatsbyNode | GatsbyIterable +type Data = IGatsbyNode | GatsbyIterable + +type OmitUndefined = (data: Data) => Partial /** * @param {Object|Array} data * @returns {Object|Array} data without undefined values */ -type omitUndefined = (data: data) => Partial - -const omitUndefined: omitUndefined = data => { +const omitUndefined: OmitUndefined = data => { const isPlainObject = _.isPlainObject(data) if (isPlainObject) { return _.pickBy(data, p => p !== undefined) @@ -20,12 +20,12 @@ const omitUndefined: omitUndefined = data => { return (data as GatsbyIterable).filter(p => p !== undefined) } +type isTypeSupported = (data: Data) => boolean + /** * @param {*} data - * @return {boolean} + * @return {boolean} Boolean if type is supported */ -type isTypeSupported = (data: data) => boolean - const isTypeSupported: isTypeSupported = data => { if (data === null) { return true @@ -41,42 +41,67 @@ const isTypeSupported: isTypeSupported = data => { return isSupported } +type sanitizeNode = ( + data: Data, + isNode?: boolean, + path?: Set +) => Data | undefined + /** * Make data serializable * @param {(Object|Array)} data to sanitize * @param {boolean} isNode = true * @param {Set} path = new Set */ - -type sanitizeNode = ( - data: data, - isNode?: boolean, - path?: Set -) => data | undefined - -const sanitizeNode: sanitizeNode = (data, isNode = true, path = new Set()) => { +export const sanitizeNode: sanitizeNode = ( + data, + isNode = true, + path = new Set() +) => { const isPlainObject = _.isPlainObject(data) + const isArray = _.isArray(data) - if (isPlainObject || _.isArray(data)) { + if (isPlainObject || isArray) { if (path.has(data)) return data path.add(data) - const returnData = isPlainObject ? {} : [] + const returnData = isPlainObject + ? ({} as IGatsbyNode) + : ([] as Array) let anyFieldChanged = false - _.each(data, (o, key) => { + + // _.each is a "Collection" method and thus objects with "length" property are iterated as arrays + const hasLengthProperty = isPlainObject + ? Object.prototype.hasOwnProperty.call(data, `length`) + : false + let lengthProperty + if (hasLengthProperty) { + lengthProperty = (data as IGatsbyNode).length + delete (data as IGatsbyNode).length + } + + _.each(data, (value, key) => { if (isNode && key === `internal`) { - returnData[key] = o + returnData[key] = value return } - returnData[key] = sanitizeNode(o as data, false, path) + returnData[key] = sanitizeNode(value as Data, false, path) - if (returnData[key] !== o) { + if (returnData[key] !== value) { anyFieldChanged = true } }) + if (hasLengthProperty) { + ;(data as IGatsbyNode).length = lengthProperty + returnData.length = sanitizeNode(lengthProperty as Data, false, path) + if (returnData.length !== lengthProperty) { + anyFieldChanged = true + } + } + if (anyFieldChanged) { - data = omitUndefined(returnData as data) as data + data = omitUndefined(returnData as Data) as Data } // arrays and plain objects are supported - no need to to sanitize @@ -89,5 +114,3 @@ const sanitizeNode: sanitizeNode = (data, isNode = true, path = new Set()) => { return data } } - -export default sanitizeNode From 0efdcd72f5b1c6e8a687610fcadd9e5569c57d74 Mon Sep 17 00:00:00 2001 From: LekoArts Date: Thu, 24 Nov 2022 15:17:19 +0100 Subject: [PATCH 6/6] add test --- .../src/utils/__tests__/sanitize-node.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/gatsby/src/utils/__tests__/sanitize-node.ts b/packages/gatsby/src/utils/__tests__/sanitize-node.ts index 429bb825d0d0a..1e91e45912fcd 100644 --- a/packages/gatsby/src/utils/__tests__/sanitize-node.ts +++ b/packages/gatsby/src/utils/__tests__/sanitize-node.ts @@ -1,4 +1,4 @@ -import sanitizeNode from "../sanitize-node" +import { sanitizeNode } from "../sanitize-node" describe(`node sanitization`, () => { let testNode @@ -90,4 +90,24 @@ describe(`node sanitization`, () => { // should be same instance expect(result).toBe(testNodeWithoutUnserializableData) }) + + it(`keeps length field but not OOM`, () => { + const testNodeWithLength = { + id: `id2`, + parent: ``, + children: [], + length: 81185414, + foo: `bar`, + internal: { + type: `Test`, + contentDigest: `digest1`, + owner: `test`, + counter: 0, + }, + fields: [], + } + const result = sanitizeNode(testNodeWithLength) + // @ts-ignore - Just for tests + expect(result.length).toBeDefined() + }) })