From e28e57bb7aa6ddc6133f6d7a4a3c83daf46d6f96 Mon Sep 17 00:00:00 2001 From: Christian Bewernitz Date: Sun, 21 Mar 2021 19:41:59 +0100 Subject: [PATCH] test(stryker): Replace line numbers by error index so the snapshots match even when stryker mutates the files. --- .gitignore | 1 + .../reported-levels.test.js.snap | 86 +++++++++---------- test/error/reported-levels.test.js | 20 ++++- test/error/reported.js | 69 +++++++++++++++ 4 files changed, 130 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index 92c4f3ff2..faab12f0c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /node_modules /reports +/test/error/reported.json diff --git a/test/error/__snapshots__/reported-levels.test.js.snap b/test/error/__snapshots__/reported-levels.test.js.snap index 529e08d69..c32c545ab 100644 --- a/test/error/__snapshots__/reported-levels.test.js.snap +++ b/test/error/__snapshots__/reported-levels.test.js.snap @@ -10,7 +10,7 @@ Array [ exports[`SYNTAX_AttributeEqualMissingValue with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: attribute value missed!!||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -24,7 +24,7 @@ Array [ exports[`SYNTAX_AttributeEqualMissingValue with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: attribute value missed!!||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -38,7 +38,7 @@ Array [ exports[`SYNTAX_AttributeMissingEndingQuote with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: attribute value no end '\\"' match||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -52,7 +52,7 @@ Array [ exports[`SYNTAX_AttributeMissingEndingQuote with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: attribute value no end '\\"' match||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -66,7 +66,7 @@ Array [ exports[`SYNTAX_ElementClosingNotConnected with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: elements closed character '/' and '>' must be connected to||@#[line:1,col:6] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -80,7 +80,7 @@ Array [ exports[`SYNTAX_ElementClosingNotConnected with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: elements closed character '/' and '>' must be connected to||@#[line:1,col:6] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -94,9 +94,9 @@ Array [ exports[`SYNTAX_EndTagMaybeNotComplete with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] end tag name: inner maybe not complete||@#[line:1,col:6] - at parse (lib/sax.js:128)", + at parse (lib/sax.js:#2)", "[xmldom error] element parse error: Error: [xmldom error] end tag name: inner maybe not complete||@#[line:1,col:6]||@#[line:1,col:6] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -110,9 +110,9 @@ Array [ exports[`SYNTAX_EndTagMaybeNotComplete with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] end tag name: inner maybe not complete||@#[line:1,col:6] - at parse (lib/sax.js:128)", + at parse (lib/sax.js:#2)", "[xmldom error] element parse error: Error: [xmldom error] end tag name: inner maybe not complete||@#[line:1,col:6]||@#[line:1,col:6] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -126,9 +126,9 @@ Array [ exports[`SYNTAX_EndTagNotComplete with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] end tag name: xml is not complete:xml||@#[line:1,col:1] - at parse (lib/sax.js:124)", + at parse (lib/sax.js:#1)", "[xmldom error] element parse error: Error: [xmldom error] end tag name: xml is not complete:xml||@#[line:1,col:1]||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -142,9 +142,9 @@ Array [ exports[`SYNTAX_EndTagNotComplete with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] end tag name: xml is not complete:xml||@#[line:1,col:1] - at parse (lib/sax.js:124)", + at parse (lib/sax.js:#1)", "[xmldom error] element parse error: Error: [xmldom error] end tag name: xml is not complete:xml||@#[line:1,col:1]||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -158,7 +158,7 @@ Array [ exports[`SYNTAX_InvalidAttributeName with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: invalid attribute:123||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -172,7 +172,7 @@ Array [ exports[`SYNTAX_InvalidAttributeName with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: invalid attribute:123||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -186,7 +186,7 @@ Array [ exports[`SYNTAX_InvalidTagName with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: invalid tagName:123||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -200,7 +200,7 @@ Array [ exports[`SYNTAX_InvalidTagName with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] element parse error: Error: invalid tagName:123||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -214,9 +214,9 @@ Array [ exports[`SYNTAX_UnclosedComment with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] Unclosed comment||@#[line:1,col:1] - at parseDCC (lib/sax.js:538)", + at parseDCC (lib/sax.js:#21)", "[xmldom error] element parse error: Error: [xmldom error] Unclosed comment||@#[line:1,col:1]||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -230,9 +230,9 @@ Array [ exports[`SYNTAX_UnclosedComment with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] Unclosed comment||@#[line:1,col:1] - at parseDCC (lib/sax.js:538)", + at parseDCC (lib/sax.js:#21)", "[xmldom error] element parse error: Error: [xmldom error] Unclosed comment||@#[line:1,col:1]||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -246,9 +246,9 @@ Array [ exports[`SYNTAX_UnexpectedEndOfInput with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] unexpected end of input||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:308)", + at parseElementStartPart (lib/sax.js:#13)", "[xmldom error] element parse error: Error: [xmldom error] unexpected end of input||@#[line:1,col:1]||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -262,9 +262,9 @@ Array [ exports[`SYNTAX_UnexpectedEndOfInput with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] unexpected end of input||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:308)", + at parseElementStartPart (lib/sax.js:#13)", "[xmldom error] element parse error: Error: [xmldom error] unexpected end of input||@#[line:1,col:1]||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -278,7 +278,7 @@ Array [ exports[`WF_AttributeMissingQuote with mimeType text/html should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"value\\" missed quot(\\")!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:333)", + at parseElementStartPart (lib/sax.js:#14)", ] `; @@ -292,7 +292,7 @@ Array [ exports[`WF_AttributeMissingQuote with mimeType text/xml should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"value\\" missed quot(\\")!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:333)", + at parseElementStartPart (lib/sax.js:#14)", ] `; @@ -306,7 +306,7 @@ Array [ exports[`WF_AttributeMissingQuote2 with mimeType text/html should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"&\\" missed quot(\\")!!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:363)", + at parseElementStartPart (lib/sax.js:#17)", ] `; @@ -320,7 +320,7 @@ Array [ exports[`WF_AttributeMissingQuote2 with mimeType text/xml should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"&\\" missed quot(\\")!!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:363)", + at parseElementStartPart (lib/sax.js:#17)", ] `; @@ -334,7 +334,7 @@ Array [ exports[`WF_AttributeMissingStartingQuote with mimeType text/html should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"attr\\" missed start quot(\\")!!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:281)", + at parseElementStartPart (lib/sax.js:#10)", ] `; @@ -348,7 +348,7 @@ Array [ exports[`WF_AttributeMissingStartingQuote with mimeType text/xml should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"attr\\" missed start quot(\\")!!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:281)", + at parseElementStartPart (lib/sax.js:#10)", ] `; @@ -362,7 +362,7 @@ Array [ exports[`WF_AttributeMissingValue with mimeType text/html should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"attr\\" missed value!! \\"attr\\" instead!!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:337)", + at parseElementStartPart (lib/sax.js:#15)", ] `; @@ -376,7 +376,7 @@ Array [ exports[`WF_AttributeMissingValue with mimeType text/xml should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"attr\\" missed value!! \\"attr\\" instead!!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:337)", + at parseElementStartPart (lib/sax.js:#15)", ] `; @@ -392,7 +392,7 @@ Array [ exports[`WF_AttributeMissingValue2 with mimeType text/html should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"attr\\" missed value!! \\"attr\\" instead2!!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:385)", + at parseElementStartPart (lib/sax.js:#18)", ] `; @@ -408,7 +408,7 @@ Array [ exports[`WF_AttributeMissingValue2 with mimeType text/xml should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute \\"attr\\" missed value!! \\"attr\\" instead2!!||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:385)", + at parseElementStartPart (lib/sax.js:#18)", ] `; @@ -422,7 +422,7 @@ Array [ exports[`WF_AttributeValueMustAfterEqual with mimeType text/html should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute value must after \\"=\\"||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:263)", + at parseElementStartPart (lib/sax.js:#8)", ] `; @@ -436,7 +436,7 @@ Array [ exports[`WF_AttributeValueMustAfterEqual with mimeType text/xml should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] attribute value must after \\"=\\"||@#[line:1,col:1] - at parseElementStartPart (lib/sax.js:263)", + at parseElementStartPart (lib/sax.js:#8)", ] `; @@ -464,9 +464,9 @@ Array [ exports[`WF_EntityDeclared with mimeType text/html should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] entity not found:&e;||@#[line:1,col:1] - at replace (lib/sax.js:71)", + at replace (lib/sax.js:#0)", "[xmldom error] element parse error: Error: [xmldom error] entity not found:&e;||@#[line:1,col:1]||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -480,9 +480,9 @@ Array [ exports[`WF_EntityDeclared with mimeType text/xml should not catch Error thrown in errorHandler.error 1`] = ` Array [ "[xmldom error] entity not found:&e;||@#[line:1,col:1] - at replace (lib/sax.js:71)", + at replace (lib/sax.js:#0)", "[xmldom error] element parse error: Error: [xmldom error] entity not found:&e;||@#[line:1,col:1]||@#[line:1,col:1] - at parse (lib/sax.js:208)", + at parse (lib/sax.js:#5)", ] `; @@ -496,6 +496,6 @@ Array [ exports[`WF_UnclosedXmlAttribute with mimeType text/xml should escalate Error thrown in errorHandler.warning to errorHandler.error 1`] = ` Array [ "[xmldom warning] unclosed xml attribute||@#[line:1,col:1] - at parse (lib/sax.js:173)", + at parse (lib/sax.js:#4)", ] `; diff --git a/test/error/reported-levels.test.js b/test/error/reported-levels.test.js index 63ba05572..5e5cc74a9 100644 --- a/test/error/reported-levels.test.js +++ b/test/error/reported-levels.test.js @@ -1,6 +1,6 @@ 'use strict' -const { REPORTED } = require('./reported') +const { LINE_TO_ERROR_INDEX, REPORTED } = require('./reported') const { getTestParser } = require('../get-test-parser') const { ParseError } = require('../../lib/sax') const { DOMParser } = require('../../lib/dom-parser') @@ -90,8 +90,10 @@ describe.each(Object.entries(REPORTED))( * - put's the message on one line as first line * - picks the first line in the stack trace that is in `lib/sax.js`, * and strips absolute paths and character position from that stack entry - * as second line + * as second line. the line number in the stack is converted to the error index + * (to make snapshot testing possible even with stryker). * @param {Error} error + * @returns {string} */ function toErrorSnapshot(error) { const libSaxMatch = /\/.*\/(lib\/sax\.js)/ @@ -102,5 +104,17 @@ function toErrorSnapshot(error) { // strip of absolute path .replace(libSaxMatch, '$1') // strip of position of character in line - .replace(/:\d+\)$/, ')')}` + .replace(/:\d+\)$/, ')') + // since stryker mutates the code, line numbers in the stack trace will change + // and no longer match the ones in the snapshots. + // So we map line numbers to an "error index" meaning "the nth thing reported by lib/sax.js" + // `./reported.js` creates that index on every test run + // and writes it `./report.json` on every run, for later inspection. + .replace(/(lib\/sax\.js:)\d+/, (fileAndLine, file) => { + return `${file}#${ + fileAndLine in LINE_TO_ERROR_INDEX + ? LINE_TO_ERROR_INDEX[fileAndLine].index + : -1 + }` + })}` } diff --git a/test/error/reported.js b/test/error/reported.js index 2820684e8..2661e92e0 100644 --- a/test/error/reported.js +++ b/test/error/reported.js @@ -1,4 +1,5 @@ 'use strict' +const fs = require('fs') /** * @typedef ErrorReport @@ -238,6 +239,74 @@ const REPORTED = { }, } +const LINE_TO_ERROR_INDEX = { + '': `This file is gitignored and is generated by ${__filename} every time the tests run.` +} + +;(function parseErrorLines() { + let errorIndex = 0 + const source = fs + .readFileSync(__dirname + '/../../lib/sax.js', 'utf8') + .split('\n') + source.forEach((lineFull, lineNumber) => { + const line = lineFull.trim() + if (/^(\/\/|\/\*|\* ?)/.test(line)) { + // ignoring single or multiline comments + return + } + if (/^(ParseError\.prototype|function ParseError)/.test(line)) { + // ignoring ParseError "class" definition + return + } + const match = /(warning|[\w.]*error)\((.*)\)/i.exec(line) + + // ignore lines that don't throw or report an error or warning + if (!match) return + + const [, errorType, message] = match + + // ignore lines that contain console.error, + // sometimes happened even when they have been commented out, + // when stryker puts code on the line before the comment + if (errorType.startsWith('console.')) return + + // the first line is line 1, not line 0! + LINE_TO_ERROR_INDEX[`lib/sax.js:${lineNumber + 1}`] = { + errorType, + index: errorIndex++, + line, + message, + } + }) + Object.entries(REPORTED).forEach(([key, value]) => { + const matches = source.reduce((lines, currentLine, i) => { + if ( + new RegExp(value.level, 'i').test(currentLine) && + value.match(currentLine) + ) { + // the first line is line 1, not line 0! + lines.push(i + 1) + } + return lines + }, []) + if (matches.length === 0) + throw `${key} doesn't match any line in lib/sax.js` + if (matches.length > 1) throw `${key} matches multiple lines in lib/sax.js` + const lineKey = `lib/sax.js:${matches[0]}` + if (!(lineKey in LINE_TO_ERROR_INDEX)) { + console.error('line not mapped:', lineKey, matches[0]) + } else { + LINE_TO_ERROR_INDEX[lineKey].reportedAs = key + } + }) + fs.writeFileSync( + `${__dirname}/reported.json`, + JSON.stringify(LINE_TO_ERROR_INDEX, null, 2), + 'utf8' + ) +})() + module.exports = { + LINE_TO_ERROR_INDEX, REPORTED, }