From 6c8eee2a22b99f3088ec5833a304d62de2f7ec3a Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 20 Sep 2020 23:22:45 +0300 Subject: [PATCH 01/89] Add TokenStream --- src/stream/token-stream.test.ts | 15 + src/stream/token-stream.ts | 479 ++++++++++++++++++++++++++++++++ tsconfig.json | 12 + 3 files changed, 506 insertions(+) create mode 100644 src/stream/token-stream.test.ts create mode 100644 src/stream/token-stream.ts create mode 100644 tsconfig.json diff --git a/src/stream/token-stream.test.ts b/src/stream/token-stream.test.ts new file mode 100644 index 00000000..3917b8d8 --- /dev/null +++ b/src/stream/token-stream.test.ts @@ -0,0 +1,15 @@ +import { TokenStream } from './token-stream.js' + +export function test(src: string) { + const tokens: string[] = [] + const ts = new TokenStream() + .on('data', chunk => { + tokens.push(chunk) + }) + .on('error', error => { + throw error + }) + ts.write(src) + ts.end() + return tokens +} diff --git a/src/stream/token-stream.ts b/src/stream/token-stream.ts new file mode 100644 index 00000000..0ff3a8fe --- /dev/null +++ b/src/stream/token-stream.ts @@ -0,0 +1,479 @@ +/* eslint-env node */ +/* eslint-disable consistent-return */ + +/* +START -> stream + +stream + directive -> line-end -> stream + indent + line-end -> stream + [else] -> line-start + +line-end + comment -> line-end + newline -> . + input-end -> END + +line-start + doc-start -> doc + doc-end -> stream + [else] -> indent -> block-start + +block-start + seq-item-start -> block-start + explicit-key-start -> block-start + map-value-start -> block-start + [else] -> doc + +doc + line-end -> line-start + spaces -> doc + anchor -> doc + tag -> doc + flow-start -> flow -> doc + flow-end -> error -> doc + seq-item-start -> error -> doc + explicit-key-start -> error -> doc + map-value-start -> doc + alias -> doc + quote-start -> quoted-scalar -> doc + block-scalar-header -> line-end -> block-scalar(min) -> line-start + [else] -> plain-scalar(false, min) -> doc + +flow + line-end -> flow + spaces -> flow + anchor -> flow + tag -> flow + flow-start -> flow -> flow + flow-end -> . + seq-item-start -> error -> flow + explicit-key-start -> flow + map-value-start -> flow + alias -> flow + quote-start -> quoted-scalar -> flow + comma -> flow + [else] -> plain-scalar(true, 0) -> flow + +quoted-scalar + quote-end -> . + [else] -> quoted-scalar + +block-scalar(min) + newline + peek(indent < min) -> . + [else] -> block-scalar(min) + +plain-scalar(is-flow, min) + scalar-end(is-flow) -> . + peek(newline + (indent < min)) -> . + [else] -> plain-scalar(min) +*/ + +import { Transform, TransformOptions } from 'stream' +import { StringDecoder } from 'string_decoder' + +import { DOCUMENT, SCALAR } from './token-type.js' + +type State = + | 'stream' + | 'line-start' + | 'block-start' + | 'doc' + | 'flow' + | 'quoted-scalar' + | 'block-scalar' + | 'plain-scalar' + +function isEmpty(ch: string) { + switch (ch) { + case undefined: + case ' ': + case '\n': + case '\r': + case '\t': + return true + default: + return false + } +} + +const invalidFlowScalarChars = [',', '[', ']', '{', '}'] +const invalidIdentifierChars = [' ', ',', '[', ']', '{', '}', '\n', '\r', '\t'] +const isNotIdentifierChar = (ch: string) => + !ch || invalidIdentifierChars.includes(ch) + +export function prettyToken(token: string) { + if (token === DOCUMENT) return '' + if (token === SCALAR) return '' + return JSON.stringify(token) +} + +export type TokenStreamOptions = Omit< + TransformOptions, + 'decodeStrings' | 'emitClose' | 'objectMode' +> + +/** Consumes string or buffer input, emits token strings */ +export class TokenStream extends Transform { + decoder: StringDecoder + + atEnd = false + buffer = '' + flowLevel = 0 + indent = 0 + indentMore = '' + next: State | null = null + pos = 0 + + constructor(options: TokenStreamOptions = {}) { + super({ + ...options, + decodeStrings: false, + emitClose: true, + objectMode: true + }) + this.decoder = new StringDecoder(options.defaultEncoding || 'utf8') + } + + _flush(done: (error?: Error) => void) { + this.atEnd = true + let next: State | null = this.next + while (next && this.hasChars(1)) next = this.parseNext(next) + done() + } + + _transform(chunk: string | Buffer, _: any, done: (error?: Error) => void) { + if (Buffer.isBuffer(chunk)) chunk = this.decoder.write(chunk) + else if (typeof chunk !== 'string') + return done(new Error('Only string and Buffer input is accepted')) + // console.log('TS', chunk) + this.buffer = this.buffer ? this.buffer + chunk : chunk + + let next: State | null = this.next || 'stream' + while (next) next = this.parseNext(next) + done() + } + + atLineEnd() { + let i = this.pos + let ch = this.buffer[i] + while (ch === ' ') ch = this.buffer[++i] + if (!ch || ch === '#' || ch === '\n') return true + if (ch === '\r') return this.buffer[i + 1] === '\n' + return false + } + + charAt(n: number) { + return this.buffer[this.pos + n] + } + + getLine(): string | null { + let end = this.buffer.indexOf('\n', this.pos) + if (end === -1) return this.atEnd ? this.buffer.substring(this.pos) : null + if (this.buffer[end - 1] === '\r') end -= 1 + return this.buffer.substring(this.pos, end) + } + + hasChars(n: number) { + return this.pos + n <= this.buffer.length + } + + setNext(state: State) { + this.buffer = this.buffer.substring(this.pos) + this.pos = 0 + this.next = state + return null + } + + peek(n: number) { + return this.buffer.substr(this.pos, n) + } + + parseNext(next: State) { + switch (next) { + case 'stream': + return this.parseStream() + case 'line-start': + return this.parseLineStart() + case 'block-start': + return this.parseBlockStart() + case 'doc': + return this.parseDocument() + case 'flow': + return this.parseFlowCollection() + case 'quoted-scalar': + return this.parseQuotedScalar() + case 'block-scalar': + return this.parseBlockScalar() + case 'plain-scalar': + return this.parsePlainScalar() + default: + throw new Error(`Unknown state ${next}`) + } + } + + parseStream() { + const line = this.getLine() + if (line === null) return this.setNext('stream') + if (line[0] === '%') { + let dirEnd = line.indexOf(' #') + 1 + if (dirEnd === 0) dirEnd = line.length + while (line[dirEnd - 1] === ' ') dirEnd -= 1 + const n = this.pushCount(dirEnd) + this.pushSpaces() + this.pushCount(line.length - n) // possible comment + this.pushNewline() + return 'stream' + } + if (this.atLineEnd()) { + const sp = this.pushSpaces() + this.pushCount(line.length - sp) + this.pushNewline() + return 'stream' + } + this.push(DOCUMENT) + return this.parseLineStart() + } + + parseLineStart() { + const ch = this.charAt(0) + if (ch === '-' || ch === '.') { + if (!this.atEnd && !this.hasChars(4)) return this.setNext('line-start') + const s = this.peek(3) + if (s === '---' && isEmpty(this.charAt(3))) { + this.pushCount(3) + this.indent = 0 + this.indentMore = '' + return 'doc' + } else if (s === '...' && isEmpty(this.charAt(3))) { + this.pushCount(3) + return 'stream' + } + } + this.indent = this.pushSpaces() + this.indentMore = '' + return this.parseBlockStart() + } + + parseBlockStart(): 'doc' | null { + const [ch0, ch1] = this.peek(2) + if (!ch1 && !this.atEnd) return this.setNext('block-start') + if ((ch0 === '-' || ch0 === '?' || ch0 === ':') && isEmpty(ch1)) { + const start = this.pos + const n = this.pushCount(1) + this.pushSpaces() + this.indentMore += this.buffer.substr(start, n) + return this.parseBlockStart() + } + if (this.indentMore.length > 2) { + let last = this.indentMore.length - 1 + while (this.indentMore[last] === ' ') last -= 1 + if (last > 0) this.indent += last + } + return 'doc' + } + + parseDocument() { + this.pushSpaces() + const line = this.getLine() + if (line === null) return this.setNext('doc') + let n = this.pushIndicators() + switch (line[n]) { + case undefined: + case '#': + this.pushCount(line.length) + this.pushNewline() + return this.parseLineStart() + case '{': + case '[': + this.pushCount(1) + this.flowLevel = 1 + return 'flow' + case '}': + case ']': + // this is an error + this.pushCount(1) + return 'doc' + case '"': + case "'": + return this.parseQuotedScalar() + case '|': + case '>': + n += this.pushUntil(isEmpty) + n += this.pushSpaces() + this.pushCount(line.length - n) + this.pushNewline() + return this.parseBlockScalar() + default: + return this.parsePlainScalar() + } + } + + parseFlowCollection() { + while (this.pushNewline() + this.pushSpaces() > 0) {} + const line = this.getLine() + if (line === null) return this.setNext('flow') + let n = line[0] === ',' ? this.pushCount(1) + this.pushSpaces() : 0 + n += this.pushIndicators() + switch (line[n]) { + case undefined: + case '#': + this.pushCount(line.length) + this.pushNewline() + return 'flow' + case '{': + case '[': + this.pushCount(1) + this.flowLevel += 1 + return 'flow' + case '}': + case ']': + this.pushCount(1) + this.flowLevel -= 1 + return this.flowLevel ? 'flow' : 'doc' + case '"': + case "'": + return this.parseQuotedScalar() + default: + return this.parsePlainScalar() + } + } + + parseQuotedScalar() { + const quote = this.charAt(0) + let end = this.buffer.indexOf(quote, this.pos + 1) + if (quote === "'") { + while (end !== -1 && this.buffer[end + 1] === "'") + end = this.buffer.indexOf("'", end + 2) + } else { + // double-quote + while (end !== -1) { + let n = 0 + while (this.buffer[end - 1 - n] === '\\') n += 1 + if (n % 2 === 0) break + end = this.buffer.indexOf('"', end + 1) + } + } + if (end === -1) return this.setNext('quoted-scalar') + this.pushToIndex(end + 1) + return this.flowLevel ? 'flow' : 'doc' + } + + parseBlockScalar() { + const reqIndent = + this.indent > 0 ? this.indent + 1 : this.indentMore ? 1 : 0 + let i = this.pos - 1 + let ch: string + while ((ch = this.buffer[++i])) { + if (ch === '\n' && reqIndent > 0) { + let indent = 0 + let next = this.buffer[i + 1] + while (next === ' ') next = this.buffer[++indent + i + 1] + if ( + indent < reqIndent && + next !== '\n' && + !(next === '\r' && this.buffer[indent + i + 2] === '\n') + ) + break + i += indent + } + } + if (!ch && !this.atEnd) return this.setNext('block-scalar') + this.push(SCALAR) + this.pushToIndex(i + 1) + return this.parseLineStart() + } + + parsePlainScalar() { + const inFlow = this.flowLevel > 0 + const reqIndent = + this.indent > 0 ? this.indent + 1 : this.indentMore ? 1 : 0 + let i = this.pos - 1 + let ch: string + while ((ch = this.buffer[++i])) { + if (ch === '\n' && reqIndent > 0) { + let indent = 0 + while (this.buffer[i + indent + 1] === ' ') indent += 1 + if (indent < reqIndent) { + if (this.buffer[i - 1] === '\r') i -= 1 + break + } + i += indent + } else if (ch === ':') { + const next = this.buffer[i + 1] + if (isEmpty(next) || (inFlow && next === ',')) break + } else if (isEmpty(ch)) { + const next = this.buffer[i + 1] + if (next === '#' || (inFlow && invalidFlowScalarChars.includes(next))) + break + } else if (inFlow && invalidFlowScalarChars.includes(ch)) break + } + if (!ch && !this.atEnd) return this.setNext('plain-scalar') + this.push(SCALAR) + this.pushToIndex(i) + return inFlow ? 'flow' : 'doc' + } + + pushCount(n: number) { + if (n > 0) { + this.push(this.buffer.substr(this.pos, n)) + this.pos += n + return n + } + return 0 + } + + pushToIndex(i: number) { + const s = this.buffer.slice(this.pos, i) + if (s) { + this.push(s) + this.pos += s.length + return s.length + } + return 0 + } + + pushIndicators(): number { + switch (this.charAt(0)) { + case '!': + case '&': + case '*': + return ( + this.pushUntil(isNotIdentifierChar) + + this.pushSpaces() + + this.pushIndicators() + ) + case ':': + case '?': // this is an error outside flow collections + case '-': // this is an error + if (isEmpty(this.charAt(1))) { + this.indentMore += ' ' + return this.pushCount(1) + this.pushSpaces() + this.pushIndicators() + } + } + return 0 + } + + pushNewline() { + const ch = this.buffer[this.pos] + if (ch === '\n') return this.pushCount(1) + else if (ch === '\r' && this.charAt(1) === '\n') return this.pushCount(2) + else return 0 + } + + pushSpaces() { + let i = this.pos + while (this.buffer[i] === ' ') i += 1 + const n = i - this.pos + if (n > 0) { + this.push(this.buffer.substr(this.pos, n)) + this.pos = i + } + return n + } + + pushUntil(test: (ch: string) => boolean) { + let i = this.pos + let ch = this.buffer[i] + while (!test(ch)) ch = this.buffer[++i] + return this.pushToIndex(i) + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..67eca3be --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "module": "CommonJS", + "moduleResolution": "node", + "noUnusedLocals": true, + "noUnusedParameters": true, + "outDir": "lib/", + "strict": true, + "target": "ES2017" + }, + "include": ["src/**/*.ts"] +} From c36150be88dd547f2e30c3f98472a24269b64006 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 26 Sep 2020 23:42:54 +0300 Subject: [PATCH 02/89] Add token type identifier --- src/stream/token-type.ts | 82 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/stream/token-type.ts diff --git a/src/stream/token-type.ts b/src/stream/token-type.ts new file mode 100644 index 00000000..172ec76d --- /dev/null +++ b/src/stream/token-type.ts @@ -0,0 +1,82 @@ +export const DOCUMENT = '\x02' // Start of Text +export const SCALAR = '\x1f' // Unit Separator + +export type SourceTokenType = + | 'doc-mode' + | 'scalar' + | 'doc-start' + | 'doc-end' + | 'space' + | 'comment' + | 'newline' + | 'directive-line' + | 'alias' + | 'anchor' + | 'tag' + | 'seq-item-ind' + | 'explicit-key-ind' + | 'map-value-ind' + | 'flow-map-start' + | 'flow-map-end' + | 'flow-seq-start' + | 'flow-seq-end' + | 'comma' + | 'single-quoted-scalar' + | 'double-quoted-scalar' + | 'block-scalar-header' + +export function tokenType(source: string): SourceTokenType | null { + switch (source) { + case DOCUMENT: // start of doc-mode + return 'doc-mode' + case SCALAR: // next token is a scalar value + return 'scalar' + case '---': + return 'doc-start' + case '...': + return 'doc-end' + case '': + case '\n': + case '\r\n': + return 'newline' + case '-': + return 'seq-item-ind' + case '?': + return 'explicit-key-ind' + case ':': + return 'map-value-ind' + case '{': + return 'flow-map-start' + case '}': + return 'flow-map-end' + case '[': + return 'flow-seq-start' + case ']': + return 'flow-seq-end' + case ',': + return 'comma' + } + switch (source[0]) { + case ' ': + case '\t': + return 'space' + case '#': + return 'comment' + case '%': + return 'directive-line' + case '*': + return 'alias' + case '&': + return 'anchor' + case '!': + return 'tag' + case "'": + return 'single-quoted-scalar' + case '"': + return 'double-quoted-scalar' + case '|': + case '>': + return 'block-scalar-header' + } + return null +} From 0e19aa251e761e5793ea6ffdf6d48044d83af602 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 26 Sep 2020 23:43:11 +0300 Subject: [PATCH 03/89] Add DocStream --- src/stream/doc-stream.ts | 479 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 src/stream/doc-stream.ts diff --git a/src/stream/doc-stream.ts b/src/stream/doc-stream.ts new file mode 100644 index 00000000..b415f7ec --- /dev/null +++ b/src/stream/doc-stream.ts @@ -0,0 +1,479 @@ +import { Transform, TransformOptions } from 'stream' +import { prettyToken } from './token-stream' +import { SourceTokenType, tokenType } from './token-type' + +export interface SourceToken { + type: SourceTokenType + indent: number + source: string + end?: Token[] +} + +export interface ErrorToken { + type: 'error' + source: string + message: string +} + +export interface Directive { + type: 'directive' + source: string + name: string + parameters: string[] +} + +export interface Document { + type: 'document' + start: Token[] + value?: Token + end?: Token[] +} + +export interface BlockScalar { + type: 'block-scalar' + indent: number + props: Token[] + source?: string +} + +export interface BlockMap { + type: 'block-map' + indent: number + items: Array< + | { start: Token[]; key?: never; sep?: never; value?: never } + | { start: Token[]; key: Token | null; sep: Token[]; value?: Token } + > +} + +export interface BlockSequence { + type: 'block-seq' + indent: number + items: Array<{ start: Token[]; value?: Token }> +} + +export interface FlowCollection { + type: 'flow-collection' + indent: number + start: Token + items: Token[] + end?: Token +} + +export type Token = + | SourceToken + | ErrorToken + | Directive + | Document + | BlockScalar + | BlockMap + | BlockSequence + | FlowCollection + +export type DocStreamOptions = Omit< + TransformOptions, + 'decodeStrings' | 'emitClose' | 'objectMode' +> + +export class DocStream extends Transform { + /** If true, space and sequence indicators count as indentation */ + atNewLine = true + + /** If true, next token is a scalar value */ + atScalar = false + + /** Current indentation level */ + indent = 0 + + /** Top indicates the bode that's currently being built */ + stack: Token[] = [] + + /** The source of the current chunk/token, set in _transform() */ + source = '' + + /** The type of the current chunk/token, set in _transform() */ + type = '' as SourceTokenType + + constructor(options: DocStreamOptions = {}) { + super({ + ...options, + decodeStrings: false, + emitClose: true, + objectMode: true + }) + } + + _flush(done: (error?: Error) => void) { + while (this.stack.length > 0) this.pop() + done() + } + + _transform(source: string, _: any, done: (error?: Error) => void) { + this.source = source + console.log('>', prettyToken(source)) + try { + if (this.atScalar) { + this.atScalar = false + this.handleToken() + return done() + } + const type = tokenType(source) + if (!type) throw new Error(`Not a YAML token: ${source}`) + if (type === 'scalar') { + this.atNewLine = false + this.atScalar = true + this.type = 'scalar' + return done() + } + + this.type = type + this.handleToken() + switch (type) { + case 'newline': + this.atNewLine = true + this.indent = 0 + break + case 'space': + case 'seq-item-ind': + if (this.atNewLine) this.indent += source.length + break + case 'doc-mode': + break + default: + this.atNewLine = false + } + done() + } catch (error) { + done(error) + } + } + + get sourceToken() { + return { + type: this.type, + indent: this.indent, + source: this.source + } as SourceToken + } + + handleToken() { + const top = this.peek() + if (!top) return this.stream() + switch (top.type) { + case 'document': + return this.document(top) + case 'alias': + case 'scalar': + case 'single-quoted-scalar': + case 'double-quoted-scalar': + return this.scalar(top) + case 'block-scalar': + return this.blockScalar(top) + case 'block-map': + return this.blockMap(top) + case 'block-seq': + return this.blockSequence(top) + case 'flow-collection': + return this.flowCollection(top) + default: + throw new Error(`Unexpected ${top.type} token in stack`) + } + } + + peek() { + return this.stack[this.stack.length - 1] + } + + pop() { + const token = this.stack.pop() + if (!token) throw new Error('Tried to pop an empty stack') + if (this.stack.length === 0) this.push(token) + else { + const top = this.peek() + switch (top.type) { + case 'document': + top.value = token + break + case 'block-map': { + const it = top.items[top.items.length - 1] + if (it.value) top.items.push({ start: [], key: token, sep: [] }) + else if (it.sep) it.value = token + else Object.assign(it, { key: token, sep: [] }) + break + } + case 'block-seq': + top.items[top.items.length - 1].value = token + break + case 'flow-collection': + top.items.push(token) + break + default: + throw new Error(`Unexpected ${top.type} top token when popping stack`) + } + } + } + + stream() { + switch (this.type) { + case 'directive-line': { + const parts = this.source.split(/ +/) + const name = parts.shift() + this.push({ + type: 'directive', + name, + parameters: parts, + source: this.source + }) + return + } + case 'doc-end': + case 'space': + case 'comment': + case 'newline': + this.push(this.sourceToken) + return + case 'doc-mode': + case 'doc-start': { + const doc: Document = { type: 'document', start: [] } + if (this.type === 'doc-start') doc.start.push(this.sourceToken) + this.stack.push(doc) + return + } + } + const message = `Unexpected ${this.type} token in YAML stream` + this.push({ type: 'error', message, source: this.source }) + } + + document(doc: Document) { + if (doc.value) return this.lineEnd(doc) + switch (this.type) { + case 'doc-start': + case 'anchor': + case 'tag': + case 'space': + case 'comment': + case 'newline': + doc.start.push(this.sourceToken) + return + case 'doc-end': + doc.start.push(this.sourceToken) + this.pop() + return + } + const bv = this.startBlockValue() + if (bv) this.stack.push(bv) + else { + const message = `Unexpected ${this.type} token in YAML document` + this.push({ type: 'error', message, source: this.source }) + } + } + + scalar(scalar: SourceToken) { + if (this.type === 'map-value-ind') { + let sep: Token[] + if (scalar.end) { + sep = scalar.end + sep.push(this.sourceToken) + delete scalar.end + } else sep = [this.sourceToken] + const map: BlockMap = { + type: 'block-map', + indent: scalar.indent, + items: [{ start: [], key: scalar, sep }] + } + this.stack[this.stack.length - 1] = map + } else this.lineEnd(scalar) + } + + blockScalar(scalar: BlockScalar) { + switch (this.type) { + case 'space': + case 'comment': + case 'newline': + scalar.props.push(this.sourceToken) + return + case 'scalar': + scalar.source = this.source + // block-scalar source includes trailing newline + this.atNewLine = true + this.indent = 0 + this.pop() + break + default: + this.pop() + this.handleToken() + } + } + + blockMap(map: BlockMap) { + const it = map.items[map.items.length - 1] + switch (this.type) { + case 'space': + case 'comment': + case 'newline': + if (it.value) map.items.push({ start: [this.sourceToken] }) + else if (it.sep) it.sep.push(this.sourceToken) + else it.start.push(this.sourceToken) + return + } + if (this.indent >= map.indent) { + switch (this.type) { + case 'anchor': + case 'tag': + if (it.value) map.items.push({ start: [this.sourceToken] }) + else if (it.sep) it.sep.push(this.sourceToken) + else it.start.push(this.sourceToken) + return + + case 'explicit-key-ind': + if (!it.sep) it.start.push(this.sourceToken) + else if (it.value || this.indent === map.indent) + map.items.push({ start: [this.sourceToken] }) + else this.stack.push(this.startBlockValue() as BlockMap) + return + + case 'map-value-ind': + if (!it.sep) Object.assign(it, { key: null, sep: [this.sourceToken] }) + else if (it.value) + map.items.push({ start: [], key: null, sep: [this.sourceToken] }) + else if (it.sep.some(tok => tok.type === 'map-value-ind')) + this.stack.push(this.startBlockValue() as BlockMap) + else it.sep.push(this.sourceToken) + return + + case 'alias': + case 'scalar': + case 'single-quoted-scalar': + case 'double-quoted-scalar': + if (!it.sep) { + Object.assign(it, { key: this.sourceToken, sep: [] }) + return + } + // fallthrough + + default: { + const bv = this.startBlockValue() + if (bv) return this.stack.push(bv) + } + } + } + this.pop() + this.handleToken() + } + + blockSequence(seq: BlockSequence) { + const it = seq.items[seq.items.length - 1] + switch (this.type) { + case 'space': + case 'comment': + case 'newline': + if (it.value) seq.items.push({ start: [this.sourceToken] }) + else it.start.push(this.sourceToken) + return + case 'anchor': + case 'tag': + if (it.value || this.indent <= seq.indent) break + it.start.push(this.sourceToken) + return + case 'seq-item-ind': + if (this.indent !== seq.indent) break + if (it.value) seq.items.push({ start: [this.sourceToken] }) + else it.start.push(this.sourceToken) + return + } + if (this.indent > seq.indent) { + const bv = this.startBlockValue() + if (bv) return this.stack.push(bv) + } + this.pop() + this.handleToken() + } + + flowCollection(fc: FlowCollection) { + switch (this.type) { + case 'space': + case 'comment': + case 'newline': + case 'comma': + case 'explicit-key-ind': + case 'map-value-ind': + case 'anchor': + case 'tag': + case 'alias': + case 'scalar': + case 'single-quoted-scalar': + case 'double-quoted-scalar': + fc.items.push(this.sourceToken) + return + + case 'flow-map-end': + case 'flow-seq-end': + fc.end = this.sourceToken + this.pop() + return + } + const bv = this.startBlockValue() + if (bv) return this.stack.push(bv) + this.pop() + this.handleToken() + } + + startBlockValue() { + const st = this.sourceToken + switch (this.type) { + case 'alias': + case 'scalar': + case 'single-quoted-scalar': + case 'double-quoted-scalar': + return st + case 'block-scalar-header': + return { + type: 'block-scalar', + indent: this.indent, + props: [st] + } as BlockScalar + case 'flow-map-start': + case 'flow-seq-start': + return { + type: 'flow-collection', + indent: this.indent, + start: st, + items: [] + } as FlowCollection + case 'seq-item-ind': + return { + type: 'block-seq', + indent: this.indent, + items: [{ start: [st] }] + } as BlockSequence + case 'explicit-key-ind': + return { + type: 'block-map', + indent: this.indent, + items: [{ start: [st] }] + } as BlockMap + case 'map-value-ind': + return { + type: 'block-map', + indent: this.indent, + items: [{ start: [], key: null, sep: [st] }] + } as BlockMap + } + return null + } + + lineEnd(token: SourceToken | Document) { + switch (this.type) { + case 'space': + case 'comment': + case 'newline': + if (token.end) token.end.push(this.sourceToken) + else token.end = [this.sourceToken] + if (this.type === 'newline') this.pop() + return + default: + this.pop() + this.handleToken() + return + } + } +} From 3846ac8a444f92dda6b13fd1f14c13b8a7c9d339 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 27 Sep 2020 09:58:08 +0300 Subject: [PATCH 04/89] Refactor: Lexer -> Parser -> ParseStream --- src/stream/{token-stream.ts => lexer.ts} | 65 ++++------- src/stream/parse-stream.ts | 39 +++++++ src/stream/{doc-stream.ts => parser.ts} | 138 +++++++++++------------ src/stream/test.ts | 13 +++ src/stream/token-stream.test.ts | 15 --- src/stream/token-type.ts | 6 + 6 files changed, 145 insertions(+), 131 deletions(-) rename src/stream/{token-stream.ts => lexer.ts} (88%) create mode 100644 src/stream/parse-stream.ts rename src/stream/{doc-stream.ts => parser.ts} (83%) create mode 100644 src/stream/test.ts delete mode 100644 src/stream/token-stream.test.ts diff --git a/src/stream/token-stream.ts b/src/stream/lexer.ts similarity index 88% rename from src/stream/token-stream.ts rename to src/stream/lexer.ts index 0ff3a8fe..d9277b84 100644 --- a/src/stream/token-stream.ts +++ b/src/stream/lexer.ts @@ -1,6 +1,3 @@ -/* eslint-env node */ -/* eslint-disable consistent-return */ - /* START -> stream @@ -69,9 +66,6 @@ plain-scalar(is-flow, min) [else] -> plain-scalar(min) */ -import { Transform, TransformOptions } from 'stream' -import { StringDecoder } from 'string_decoder' - import { DOCUMENT, SCALAR } from './token-type.js' type State = @@ -102,20 +96,8 @@ const invalidIdentifierChars = [' ', ',', '[', ']', '{', '}', '\n', '\r', '\t'] const isNotIdentifierChar = (ch: string) => !ch || invalidIdentifierChars.includes(ch) -export function prettyToken(token: string) { - if (token === DOCUMENT) return '' - if (token === SCALAR) return '' - return JSON.stringify(token) -} - -export type TokenStreamOptions = Omit< - TransformOptions, - 'decodeStrings' | 'emitClose' | 'objectMode' -> - -/** Consumes string or buffer input, emits token strings */ -export class TokenStream extends Transform { - decoder: StringDecoder +export class Lexer { + push: (token: string) => void atEnd = false buffer = '' @@ -125,33 +107,28 @@ export class TokenStream extends Transform { next: State | null = null pos = 0 - constructor(options: TokenStreamOptions = {}) { - super({ - ...options, - decodeStrings: false, - emitClose: true, - objectMode: true - }) - this.decoder = new StringDecoder(options.defaultEncoding || 'utf8') - } - - _flush(done: (error?: Error) => void) { - this.atEnd = true - let next: State | null = this.next - while (next && this.hasChars(1)) next = this.parseNext(next) - done() + /** + * Define/initialise a YAML lexer. `push` will be called separately with each + * token when `lex()` is passed an input string. + * + * @public + */ + constructor(push: (token: string) => void) { + this.push = push } - _transform(chunk: string | Buffer, _: any, done: (error?: Error) => void) { - if (Buffer.isBuffer(chunk)) chunk = this.decoder.write(chunk) - else if (typeof chunk !== 'string') - return done(new Error('Only string and Buffer input is accepted')) - // console.log('TS', chunk) - this.buffer = this.buffer ? this.buffer + chunk : chunk - + /** + * Read YAML tokens from the `source` string, calling the callback + * defined in the constructor for each one. If `incomplete`, a part + * of the last line may be left as a buffer for the next call. + * + * @public + */ + lex(source: string, incomplete: boolean) { + if (source) this.buffer = this.buffer ? this.buffer + source : source + this.atEnd = !incomplete let next: State | null = this.next || 'stream' - while (next) next = this.parseNext(next) - done() + while (next && (incomplete || this.hasChars(1))) next = this.parseNext(next) } atLineEnd() { diff --git a/src/stream/parse-stream.ts b/src/stream/parse-stream.ts new file mode 100644 index 00000000..52d11d77 --- /dev/null +++ b/src/stream/parse-stream.ts @@ -0,0 +1,39 @@ +import { Transform, TransformOptions } from 'stream' +import { StringDecoder } from 'string_decoder' +import { Parser } from './parser.js' + +export type ParseStreamOptions = Omit< + TransformOptions, + 'decodeStrings' | 'emitClose' | 'objectMode' +> + +export class ParseStream extends Transform { + decoder: StringDecoder + parser: Parser + + constructor(options: ParseStreamOptions = {}) { + super({ + ...options, + decodeStrings: false, + emitClose: true, + objectMode: true + }) + this.decoder = new StringDecoder(options.defaultEncoding || 'utf8') + this.parser = new Parser(token => this.push(token)) + } + + _flush(done: (error?: Error) => void) { + this.parser.parse('', false) + done() + } + + _transform(chunk: string | Buffer, _: any, done: (error?: Error) => void) { + try { + const src = Buffer.isBuffer(chunk) ? this.decoder.write(chunk) : chunk + this.parser.parse(src, true) + done() + } catch (error) { + done(error) + } + } +} diff --git a/src/stream/doc-stream.ts b/src/stream/parser.ts similarity index 83% rename from src/stream/doc-stream.ts rename to src/stream/parser.ts index b415f7ec..cf6f6307 100644 --- a/src/stream/doc-stream.ts +++ b/src/stream/parser.ts @@ -1,6 +1,5 @@ -import { Transform, TransformOptions } from 'stream' -import { prettyToken } from './token-stream' -import { SourceTokenType, tokenType } from './token-type' +import { Lexer } from './lexer' +import { SourceTokenType, prettyToken, tokenType } from './token-type' export interface SourceToken { type: SourceTokenType @@ -18,8 +17,6 @@ export interface ErrorToken { export interface Directive { type: 'directive' source: string - name: string - parameters: string[] } export interface Document { @@ -69,12 +66,12 @@ export type Token = | BlockSequence | FlowCollection -export type DocStreamOptions = Omit< - TransformOptions, - 'decodeStrings' | 'emitClose' | 'objectMode' -> +/** A YAML concrete syntax tree parser */ +export class Parser { + push: (token: Token) => void + + lexer = new Lexer(ts => this.token(ts)) -export class DocStream extends Transform { /** If true, space and sequence indicators count as indentation */ atNewLine = true @@ -87,63 +84,68 @@ export class DocStream extends Transform { /** Top indicates the bode that's currently being built */ stack: Token[] = [] - /** The source of the current chunk/token, set in _transform() */ + /** The source of the current token, set in parse() */ source = '' - /** The type of the current chunk/token, set in _transform() */ + /** The type of the current token, set in parse() */ type = '' as SourceTokenType - constructor(options: DocStreamOptions = {}) { - super({ - ...options, - decodeStrings: false, - emitClose: true, - objectMode: true - }) +/** + * @param push - Called separately with each parsed token + * @public + */ + constructor(push: (token: Token) => void) { + this.push = push } - _flush(done: (error?: Error) => void) { - while (this.stack.length > 0) this.pop() - done() + /** + * Parse `source` as YAML, calling the constructor's callback once each + * directive, document and other structure is completely parsed. If `incomplete`, + * a part of the last line may be left as a buffer for the next call. + * + * May throw on really unexpected errors. + * + * @public + */ + parse(source: string, incomplete = false) { + this.lexer.lex(source, incomplete) + if (!incomplete) while (this.stack.length > 0) this.pop() } - _transform(source: string, _: any, done: (error?: Error) => void) { + /** Advance the parser by the `source` of one lexical token. */ + token(source: string) { this.source = source console.log('>', prettyToken(source)) - try { - if (this.atScalar) { - this.atScalar = false - this.handleToken() - return done() - } - const type = tokenType(source) - if (!type) throw new Error(`Not a YAML token: ${source}`) - if (type === 'scalar') { - this.atNewLine = false - this.atScalar = true - this.type = 'scalar' - return done() - } - this.type = type - this.handleToken() - switch (type) { - case 'newline': - this.atNewLine = true - this.indent = 0 - break - case 'space': - case 'seq-item-ind': - if (this.atNewLine) this.indent += source.length - break - case 'doc-mode': - break - default: - this.atNewLine = false - } - done() - } catch (error) { - done(error) + if (this.atScalar) { + this.atScalar = false + this.step() + return + } + const type = tokenType(source) + if (!type) throw new Error(`Not a YAML token: ${source}`) + if (type === 'scalar') { + this.atNewLine = false + this.atScalar = true + this.type = 'scalar' + return + } + + this.type = type + this.step() + switch (type) { + case 'newline': + this.atNewLine = true + this.indent = 0 + break + case 'space': + case 'seq-item-ind': + if (this.atNewLine) this.indent += source.length + break + case 'doc-mode': + break + default: + this.atNewLine = false } } @@ -155,7 +157,7 @@ export class DocStream extends Transform { } as SourceToken } - handleToken() { + step() { const top = this.peek() if (!top) return this.stream() switch (top.type) { @@ -214,17 +216,9 @@ export class DocStream extends Transform { stream() { switch (this.type) { - case 'directive-line': { - const parts = this.source.split(/ +/) - const name = parts.shift() - this.push({ - type: 'directive', - name, - parameters: parts, - source: this.source - }) + case 'directive-line': + this.push({ type: 'directive', source: this.source }) return - } case 'doc-end': case 'space': case 'comment': @@ -300,7 +294,7 @@ export class DocStream extends Transform { break default: this.pop() - this.handleToken() + this.step() } } @@ -357,7 +351,7 @@ export class DocStream extends Transform { } } this.pop() - this.handleToken() + this.step() } blockSequence(seq: BlockSequence) { @@ -385,7 +379,7 @@ export class DocStream extends Transform { if (bv) return this.stack.push(bv) } this.pop() - this.handleToken() + this.step() } flowCollection(fc: FlowCollection) { @@ -414,7 +408,7 @@ export class DocStream extends Transform { const bv = this.startBlockValue() if (bv) return this.stack.push(bv) this.pop() - this.handleToken() + this.step() } startBlockValue() { @@ -472,7 +466,7 @@ export class DocStream extends Transform { return default: this.pop() - this.handleToken() + this.step() return } } diff --git a/src/stream/test.ts b/src/stream/test.ts new file mode 100644 index 00000000..74ad512f --- /dev/null +++ b/src/stream/test.ts @@ -0,0 +1,13 @@ +import { ParseStream } from './parse-stream.js' +import { Parser } from './parser.js' + +export function stream(source: string) { + const ps = new ParseStream().on('data', d => console.dir(d, { depth: null })) + ps.write(source) + ps.end() +} + +export function sync(source: string) { + const parser = new Parser(t => console.dir(t, { depth: null })) + parser.parse(source) +} diff --git a/src/stream/token-stream.test.ts b/src/stream/token-stream.test.ts deleted file mode 100644 index 3917b8d8..00000000 --- a/src/stream/token-stream.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { TokenStream } from './token-stream.js' - -export function test(src: string) { - const tokens: string[] = [] - const ts = new TokenStream() - .on('data', chunk => { - tokens.push(chunk) - }) - .on('error', error => { - throw error - }) - ts.write(src) - ts.end() - return tokens -} diff --git a/src/stream/token-type.ts b/src/stream/token-type.ts index 172ec76d..156ef8a0 100644 --- a/src/stream/token-type.ts +++ b/src/stream/token-type.ts @@ -25,6 +25,12 @@ export type SourceTokenType = | 'double-quoted-scalar' | 'block-scalar-header' +export function prettyToken(token: string) { + if (token === DOCUMENT) return '' + if (token === SCALAR) return '' + return JSON.stringify(token) +} + export function tokenType(source: string): SourceTokenType | null { switch (source) { case DOCUMENT: // start of doc-mode From d258b35764e08c68c6ae11512ea611c902f33933 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 27 Sep 2020 12:24:26 +0300 Subject: [PATCH 05/89] Push rather than throw errors --- src/stream/lexer.ts | 2 - src/stream/parse-stream.ts | 2 +- src/stream/parser.ts | 105 ++++++++++++++++++++++--------------- 3 files changed, 63 insertions(+), 46 deletions(-) diff --git a/src/stream/lexer.ts b/src/stream/lexer.ts index d9277b84..253458e1 100644 --- a/src/stream/lexer.ts +++ b/src/stream/lexer.ts @@ -184,8 +184,6 @@ export class Lexer { return this.parseBlockScalar() case 'plain-scalar': return this.parsePlainScalar() - default: - throw new Error(`Unknown state ${next}`) } } diff --git a/src/stream/parse-stream.ts b/src/stream/parse-stream.ts index 52d11d77..2e634fa5 100644 --- a/src/stream/parse-stream.ts +++ b/src/stream/parse-stream.ts @@ -33,7 +33,7 @@ export class ParseStream extends Transform { this.parser.parse(src, true) done() } catch (error) { - done(error) + done(error) // should never happen } } } diff --git a/src/stream/parser.ts b/src/stream/parser.ts index cf6f6307..94fdc966 100644 --- a/src/stream/parser.ts +++ b/src/stream/parser.ts @@ -90,10 +90,10 @@ export class Parser { /** The type of the current token, set in parse() */ type = '' as SourceTokenType -/** - * @param push - Called separately with each parsed token - * @public - */ + /** + * @param push - Called separately with each parsed token + * @public + */ constructor(push: (token: Token) => void) { this.push = push } @@ -103,8 +103,7 @@ export class Parser { * directive, document and other structure is completely parsed. If `incomplete`, * a part of the last line may be left as a buffer for the next call. * - * May throw on really unexpected errors. - * + * Errors are not thrown, but pushed out as `{ type: 'error', message }` tokens. * @public */ parse(source: string, incomplete = false) { @@ -123,29 +122,30 @@ export class Parser { return } const type = tokenType(source) - if (!type) throw new Error(`Not a YAML token: ${source}`) - if (type === 'scalar') { + if (!type) { + const message = `Not a YAML token: ${source}` + this.pop({ type: 'error', source, message }) + } else if (type === 'scalar') { this.atNewLine = false this.atScalar = true this.type = 'scalar' - return - } - - this.type = type - this.step() - switch (type) { - case 'newline': - this.atNewLine = true - this.indent = 0 - break - case 'space': - case 'seq-item-ind': - if (this.atNewLine) this.indent += source.length - break - case 'doc-mode': - break - default: - this.atNewLine = false + } else { + this.type = type + this.step() + switch (type) { + case 'newline': + this.atNewLine = true + this.indent = 0 + break + case 'space': + case 'seq-item-ind': + if (this.atNewLine) this.indent += source.length + break + case 'doc-mode': + break + default: + this.atNewLine = false + } } } @@ -176,25 +176,30 @@ export class Parser { return this.blockSequence(top) case 'flow-collection': return this.flowCollection(top) - default: - throw new Error(`Unexpected ${top.type} token in stack`) } + this.pop() // error } peek() { return this.stack[this.stack.length - 1] } - pop() { - const token = this.stack.pop() - if (!token) throw new Error('Tried to pop an empty stack') - if (this.stack.length === 0) this.push(token) - else { + pop(error?: Token) { + const token = error || this.stack.pop() + if (!token) { + const message = 'Tried to pop an empty stack' + this.push({ type: 'error', source: '', message }) + } else if (this.stack.length === 0) { + this.push(token) + } else { const top = this.peek() switch (top.type) { case 'document': top.value = token break + case 'block-scalar': + top.props.push(token) // error + break case 'block-map': { const it = top.items[top.items.length - 1] if (it.value) top.items.push({ start: [], key: token, sep: [] }) @@ -202,14 +207,18 @@ export class Parser { else Object.assign(it, { key: token, sep: [] }) break } - case 'block-seq': - top.items[top.items.length - 1].value = token + case 'block-seq': { + const it = top.items[top.items.length - 1] + if (it.value) top.items.push({ start: [], value: token }) + else it.value = token break + } case 'flow-collection': top.items.push(token) break default: - throw new Error(`Unexpected ${top.type} top token when popping stack`) + this.pop() + this.pop(token) } } } @@ -300,6 +309,7 @@ export class Parser { blockMap(map: BlockMap) { const it = map.items[map.items.length - 1] + // it.sep is true-ish if pair already has key or : separator switch (this.type) { case 'space': case 'comment': @@ -322,7 +332,12 @@ export class Parser { if (!it.sep) it.start.push(this.sourceToken) else if (it.value || this.indent === map.indent) map.items.push({ start: [this.sourceToken] }) - else this.stack.push(this.startBlockValue() as BlockMap) + else + this.stack.push({ + type: 'block-map', + indent: this.indent, + items: [{ start: [this.sourceToken] }] + }) return case 'map-value-ind': @@ -330,7 +345,11 @@ export class Parser { else if (it.value) map.items.push({ start: [], key: null, sep: [this.sourceToken] }) else if (it.sep.some(tok => tok.type === 'map-value-ind')) - this.stack.push(this.startBlockValue() as BlockMap) + this.stack.push({ + type: 'block-map', + indent: this.indent, + items: [{ start: [], key: null, sep: [this.sourceToken] }] + }) else it.sep.push(this.sourceToken) return @@ -338,11 +357,11 @@ export class Parser { case 'scalar': case 'single-quoted-scalar': case 'double-quoted-scalar': - if (!it.sep) { - Object.assign(it, { key: this.sourceToken, sep: [] }) - return - } - // fallthrough + if (it.value) + map.items.push({ start: [], key: this.sourceToken, sep: [] }) + else if (it.sep) this.stack.push(this.sourceToken) + else Object.assign(it, { key: this.sourceToken, sep: [] }) + return default: { const bv = this.startBlockValue() From 17c9b4bf91b6c3355e9fb72eaa0a84641a315053 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 27 Sep 2020 21:10:17 +0300 Subject: [PATCH 06/89] lexer: Harmonise scalar unindent/doc-end detection --- src/stream/lexer.ts | 67 +++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/src/stream/lexer.ts b/src/stream/lexer.ts index 253458e1..e043a9b5 100644 --- a/src/stream/lexer.ts +++ b/src/stream/lexer.ts @@ -144,6 +144,23 @@ export class Lexer { return this.buffer[this.pos + n] } + continueScalar(offset: number, reqIndent: number) { + let ch = this.buffer[offset] + if (reqIndent > 0) { + let indent = 0 + while (ch === ' ') ch = this.buffer[++indent + offset] + if (ch === '\r' && this.buffer[indent + offset + 1] === '\n') + return offset + indent + 1 + return ch === '\n' || indent >= reqIndent ? offset + indent : -1 + } + if (ch === '-' || ch === '.') { + const dt = this.buffer.substr(offset, 3) + if ((dt === '---' || dt === '...') && isEmpty(this.buffer[offset + 3])) + return -1 + } + return offset + } + getLine(): string | null { let end = this.buffer.indexOf('\n', this.pos) if (end === -1) return this.atEnd ? this.buffer.substring(this.pos) : null @@ -327,6 +344,17 @@ export class Lexer { end = this.buffer.indexOf('"', end + 1) } } + let nl = this.buffer.indexOf('\n', this.pos) + if (nl !== -1 && nl < end) { + const reqIndent = + this.indent > 0 ? this.indent + 1 : this.indentMore ? 1 : 0 + while (nl !== -1 && nl < end) { + const cs = this.continueScalar(nl + 1, reqIndent) + if (cs === -1) break + nl = this.buffer.indexOf('\n', cs) + } + if (nl !== -1 && nl < end) end = nl - 1 + } if (end === -1) return this.setNext('quoted-scalar') this.pushToIndex(end + 1) return this.flowLevel ? 'flow' : 'doc' @@ -335,25 +363,15 @@ export class Lexer { parseBlockScalar() { const reqIndent = this.indent > 0 ? this.indent + 1 : this.indentMore ? 1 : 0 - let i = this.pos - 1 - let ch: string - while ((ch = this.buffer[++i])) { - if (ch === '\n' && reqIndent > 0) { - let indent = 0 - let next = this.buffer[i + 1] - while (next === ' ') next = this.buffer[++indent + i + 1] - if ( - indent < reqIndent && - next !== '\n' && - !(next === '\r' && this.buffer[indent + i + 2] === '\n') - ) - break - i += indent - } + let nl = this.buffer.indexOf('\n', this.pos) + while (nl !== -1) { + const cs = this.continueScalar(nl + 1, reqIndent) + if (cs === -1) break + nl = this.buffer.indexOf('\n', cs) } - if (!ch && !this.atEnd) return this.setNext('block-scalar') + if (nl === -1 && !this.atEnd) return this.setNext('block-scalar') this.push(SCALAR) - this.pushToIndex(i + 1) + this.pushToIndex(nl + 1) return this.parseLineStart() } @@ -364,21 +382,18 @@ export class Lexer { let i = this.pos - 1 let ch: string while ((ch = this.buffer[++i])) { - if (ch === '\n' && reqIndent > 0) { - let indent = 0 - while (this.buffer[i + indent + 1] === ' ') indent += 1 - if (indent < reqIndent) { - if (this.buffer[i - 1] === '\r') i -= 1 - break - } - i += indent - } else if (ch === ':') { + if (ch === ':') { const next = this.buffer[i + 1] if (isEmpty(next) || (inFlow && next === ',')) break } else if (isEmpty(ch)) { const next = this.buffer[i + 1] if (next === '#' || (inFlow && invalidFlowScalarChars.includes(next))) break + if (ch === '\n') { + const cs = this.continueScalar(i + 1, reqIndent) + if (cs === -1) break + i = Math.max(i, cs - 2) // to advance, but still account for ' #' + } } else if (inFlow && invalidFlowScalarChars.includes(ch)) break } if (!ch && !this.atEnd) return this.setNext('plain-scalar') From eebbe89807b90ed1d6eb163953ae8a1900d5e8da Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 27 Sep 2020 22:21:13 +0300 Subject: [PATCH 07/89] lexer: Fix indent handling for maps in sequences --- src/stream/lexer.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/stream/lexer.ts b/src/stream/lexer.ts index e043a9b5..e3fcf4b9 100644 --- a/src/stream/lexer.ts +++ b/src/stream/lexer.ts @@ -258,7 +258,10 @@ export class Lexer { if (this.indentMore.length > 2) { let last = this.indentMore.length - 1 while (this.indentMore[last] === ' ') last -= 1 - if (last > 0) this.indent += last + if (last > 0) { + this.indent += last + this.indentMore = this.indentMore.slice(last) + } } return 'doc' } @@ -435,7 +438,10 @@ export class Lexer { case '?': // this is an error outside flow collections case '-': // this is an error if (isEmpty(this.charAt(1))) { - this.indentMore += ' ' + if (this.indentMore) { + this.indent += this.indentMore.length + this.indentMore = '' + } return this.pushCount(1) + this.pushSpaces() + this.pushIndicators() } } From 69ca08de104612ad47c566035ceec9d0add37e9a Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 28 Sep 2020 23:09:35 +0300 Subject: [PATCH 08/89] Add some d.ts files next to their js sources, for now copying content from those in root --- src/ast/index.d.ts | 208 ++++++++++++++++++++++++++++++++++++++++++ src/constants.d.ts | 20 ++++ src/cst/index.d.ts | 182 ++++++++++++++++++++++++++++++++++++ src/doc/Document.d.ts | 169 ++++++++++++++++++++++++++++++++++ src/doc/Schema.d.ts | 168 ++++++++++++++++++++++++++++++++++ src/errors.d.ts | 44 +++++++++ src/options.d.ts | 192 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 983 insertions(+) create mode 100644 src/ast/index.d.ts create mode 100644 src/constants.d.ts create mode 100644 src/cst/index.d.ts create mode 100644 src/doc/Document.d.ts create mode 100644 src/doc/Schema.d.ts create mode 100644 src/errors.d.ts create mode 100644 src/options.d.ts diff --git a/src/ast/index.d.ts b/src/ast/index.d.ts new file mode 100644 index 00000000..46ca2a51 --- /dev/null +++ b/src/ast/index.d.ts @@ -0,0 +1,208 @@ +import { Type } from '../constants' +import { CST } from '../cst' +import { Schema } from '../doc/Schema' + +export class Node { + /** A comment on or immediately after this */ + comment?: string | null + /** A comment before this */ + commentBefore?: string | null + /** Only available when `keepCstNodes` is set to `true` */ + cstNode?: CST.Node + /** + * The [start, end] range of characters of the source parsed + * into this node (undefined for pairs or if not parsed) + */ + range?: [number, number] | null + /** A blank line before this node and its commentBefore */ + spaceBefore?: boolean + /** A fully qualified tag, if required */ + tag?: string + /** A plain JS representation of this node */ + toJSON(arg?: any): any + /** The type of this node */ + type?: Type | Pair.Type +} + +export class Scalar extends Node { + constructor(value: any) + type?: Scalar.Type + /** + * By default (undefined), numbers use decimal notation. + * The YAML 1.2 core schema only supports 'HEX' and 'OCT'. + */ + format?: 'BIN' | 'HEX' | 'OCT' | 'TIME' + value: any + toJSON(arg?: any, ctx?: AST.NodeToJsonContext): any + toString(): string +} +export namespace Scalar { + type Type = + | Type.BLOCK_FOLDED + | Type.BLOCK_LITERAL + | Type.PLAIN + | Type.QUOTE_DOUBLE + | Type.QUOTE_SINGLE +} + +export class Alias extends Node { + type: Type.ALIAS + source: Node + cstNode?: CST.Alias + toString(ctx: Schema.StringifyContext): string +} + +export class Pair extends Node { + constructor(key: any, value?: any) + type: Pair.Type.PAIR | Pair.Type.MERGE_PAIR + /** Always Node or null when parsed, but can be set to anything. */ + key: any + /** Always Node or null when parsed, but can be set to anything. */ + value: any + cstNode?: never // no corresponding cstNode + toJSON(arg?: any, ctx?: AST.NodeToJsonContext): object | Map + toString( + ctx?: Schema.StringifyContext, + onComment?: () => void, + onChompKeep?: () => void + ): string +} +export namespace Pair { + enum Type { + PAIR = 'PAIR', + MERGE_PAIR = 'MERGE_PAIR' + } +} + +export class Merge extends Pair { + type: Pair.Type.MERGE_PAIR + /** Always Scalar('<<'), defined by the type specification */ + key: AST.PlainValue + /** Always YAMLSeq, stringified as *A if length = 1 */ + value: YAMLSeq + toString(ctx?: Schema.StringifyContext, onComment?: () => void): string +} + +export class Collection extends Node { + type?: Type.MAP | Type.FLOW_MAP | Type.SEQ | Type.FLOW_SEQ | Type.DOCUMENT + items: any[] + schema?: Schema + + /** + * Adds a value to the collection. For `!!map` and `!!omap` the value must + * be a Pair instance or a `{ key, value }` object, which may not have a key + * that already exists in the map. + */ + add(value: any): void + addIn(path: Iterable, value: any): void + /** + * Removes a value from the collection. + * @returns `true` if the item was found and removed. + */ + delete(key: any): boolean + deleteIn(path: Iterable): boolean + /** + * Returns item at `key`, or `undefined` if not found. By default unwraps + * scalar values from their surrounding node; to disable set `keepScalar` to + * `true` (collections are always returned intact). + */ + get(key: any, keepScalar?: boolean): any + getIn(path: Iterable, keepScalar?: boolean): any + /** + * Checks if the collection includes a value with the key `key`. + */ + has(key: any): boolean + hasIn(path: Iterable): boolean + /** + * Sets a value in this collection. For `!!set`, `value` needs to be a + * boolean to add/remove the item from the set. + */ + set(key: any, value: any): void + setIn(path: Iterable, value: any): void +} + +export class YAMLMap extends Collection { + type?: Type.FLOW_MAP | Type.MAP + items: Array + hasAllNullValues(): boolean + toJSON(arg?: any, ctx?: AST.NodeToJsonContext): object | Map + toString( + ctx?: Schema.StringifyContext, + onComment?: () => void, + onChompKeep?: () => void + ): string +} + +export class YAMLSeq extends Collection { + type?: Type.FLOW_SEQ | Type.SEQ + delete(key: number | string | Scalar): boolean + get(key: number | string | Scalar, keepScalar?: boolean): any + has(key: number | string | Scalar): boolean + set(key: number | string | Scalar, value: any): void + hasAllNullValues(): boolean + toJSON(arg?: any, ctx?: AST.NodeToJsonContext): any[] + toString( + ctx?: Schema.StringifyContext, + onComment?: () => void, + onChompKeep?: () => void + ): string +} + +export namespace AST { + interface NodeToJsonContext { + anchors?: any[] + doc: Document + keep?: boolean + mapAsMap?: boolean + maxAliasCount?: number + onCreate?: (node: Node) => void + [key: string]: any + } + + interface BlockFolded extends Scalar { + type: Type.BLOCK_FOLDED + cstNode?: CST.BlockFolded + } + + interface BlockLiteral extends Scalar { + type: Type.BLOCK_LITERAL + cstNode?: CST.BlockLiteral + } + + interface PlainValue extends Scalar { + type: Type.PLAIN + cstNode?: CST.PlainValue + } + + interface QuoteDouble extends Scalar { + type: Type.QUOTE_DOUBLE + cstNode?: CST.QuoteDouble + } + + interface QuoteSingle extends Scalar { + type: Type.QUOTE_SINGLE + cstNode?: CST.QuoteSingle + } + + interface FlowMap extends YAMLMap { + type: Type.FLOW_MAP + cstNode?: CST.FlowMap + } + + interface BlockMap extends YAMLMap { + type: Type.MAP + cstNode?: CST.Map + } + + interface FlowSeq extends YAMLSeq { + type: Type.FLOW_SEQ + items: Array + cstNode?: CST.FlowSeq + } + + interface BlockSeq extends YAMLSeq { + type: Type.SEQ + items: Array + cstNode?: CST.Seq + } +} diff --git a/src/constants.d.ts b/src/constants.d.ts new file mode 100644 index 00000000..d5110c9c --- /dev/null +++ b/src/constants.d.ts @@ -0,0 +1,20 @@ +export enum Type { + ALIAS = 'ALIAS', + BLANK_LINE = 'BLANK_LINE', + BLOCK_FOLDED = 'BLOCK_FOLDED', + BLOCK_LITERAL = 'BLOCK_LITERAL', + COMMENT = 'COMMENT', + DIRECTIVE = 'DIRECTIVE', + DOCUMENT = 'DOCUMENT', + FLOW_MAP = 'FLOW_MAP', + FLOW_SEQ = 'FLOW_SEQ', + MAP = 'MAP', + MAP_KEY = 'MAP_KEY', + MAP_VALUE = 'MAP_VALUE', + PLAIN = 'PLAIN', + QUOTE_DOUBLE = 'QUOTE_DOUBLE', + QUOTE_SINGLE = 'QUOTE_SINGLE', + SEQ = 'SEQ', + SEQ_ITEM = 'SEQ_ITEM' +} + diff --git a/src/cst/index.d.ts b/src/cst/index.d.ts new file mode 100644 index 00000000..d67d1f18 --- /dev/null +++ b/src/cst/index.d.ts @@ -0,0 +1,182 @@ +import { Type } from '../constants' +import { YAMLSyntaxError } from '../errors' + +export namespace CST { + interface Range { + start: number + end: number + origStart?: number + origEnd?: number + isEmpty(): boolean + } + + interface ParseContext { + /** Node starts at beginning of line */ + atLineStart: boolean + /** true if currently in a collection context */ + inCollection: boolean + /** true if currently in a flow context */ + inFlow: boolean + /** Current level of indentation */ + indent: number + /** Start of the current line */ + lineStart: number + /** The parent of the node */ + parent: Node + /** Source of the YAML document */ + src: string + } + + interface Node { + context: ParseContext | null + /** if not null, indicates a parser failure */ + error: YAMLSyntaxError | null + /** span of context.src parsed into this node */ + range: Range | null + valueRange: Range | null + /** anchors, tags and comments */ + props: Range[] + /** specific node type */ + type: Type + /** if non-null, overrides source value */ + value: string | null + + readonly anchor: string | null + readonly comment: string | null + readonly hasComment: boolean + readonly hasProps: boolean + readonly jsonLike: boolean + readonly rawValue: string | null + readonly tag: + | null + | { verbatim: string } + | { handle: string; suffix: string } + readonly valueRangeContainsNewline: boolean + } + + interface Alias extends Node { + type: Type.ALIAS + /** contain the anchor without the * prefix */ + readonly rawValue: string + } + + type Scalar = BlockValue | PlainValue | QuoteValue + + interface BlockValue extends Node { + type: Type.BLOCK_FOLDED | Type.BLOCK_LITERAL + chomping: 'CLIP' | 'KEEP' | 'STRIP' + blockIndent: number | null + header: Range + readonly strValue: string | null + } + + interface BlockFolded extends BlockValue { + type: Type.BLOCK_FOLDED + } + + interface BlockLiteral extends BlockValue { + type: Type.BLOCK_LITERAL + } + + interface PlainValue extends Node { + type: Type.PLAIN + readonly strValue: string | null + } + + interface QuoteValue extends Node { + type: Type.QUOTE_DOUBLE | Type.QUOTE_SINGLE + readonly strValue: + | null + | string + | { str: string; errors: YAMLSyntaxError[] } + } + + interface QuoteDouble extends QuoteValue { + type: Type.QUOTE_DOUBLE + } + + interface QuoteSingle extends QuoteValue { + type: Type.QUOTE_SINGLE + } + + interface Comment extends Node { + type: Type.COMMENT + readonly anchor: null + readonly comment: string + readonly rawValue: null + readonly tag: null + } + + interface BlankLine extends Node { + type: Type.BLANK_LINE + } + + interface MapItem extends Node { + type: Type.MAP_KEY | Type.MAP_VALUE + node: ContentNode | null + } + + interface MapKey extends MapItem { + type: Type.MAP_KEY + } + + interface MapValue extends MapItem { + type: Type.MAP_VALUE + } + + interface Map extends Node { + type: Type.MAP + /** implicit keys are not wrapped */ + items: Array + } + + interface SeqItem extends Node { + type: Type.SEQ_ITEM + node: ContentNode | null + } + + interface Seq extends Node { + type: Type.SEQ + items: Array + } + + interface FlowChar { + char: '{' | '}' | '[' | ']' | ',' | '?' | ':' + offset: number + origOffset?: number + } + + interface FlowCollection extends Node { + type: Type.FLOW_MAP | Type.FLOW_SEQ + items: Array< + FlowChar | BlankLine | Comment | Alias | Scalar | FlowCollection + > + } + + interface FlowMap extends FlowCollection { + type: Type.FLOW_MAP + } + + interface FlowSeq extends FlowCollection { + type: Type.FLOW_SEQ + } + + type ContentNode = Alias | Scalar | Map | Seq | FlowCollection + + interface Directive extends Node { + type: Type.DIRECTIVE + name: string + readonly anchor: null + readonly parameters: string[] + readonly tag: null + } + + interface Document extends Node { + type: Type.DOCUMENT + directives: Array + contents: Array + readonly anchor: null + readonly comment: null + readonly tag: null + } +} diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts new file mode 100644 index 00000000..d90f0321 --- /dev/null +++ b/src/doc/Document.d.ts @@ -0,0 +1,169 @@ +import { Alias, Collection, Merge, Pair } from '../ast' +import { Type } from '../constants' +import { CST } from '../cst' +import { YAMLError, YAMLWarning } from '../errors' +import { Options } from '../options' +import { Schema } from './Schema' + +type Replacer = any[] | ((key: any, value: any) => boolean) +type Reviver = (key: any, value: any) => any + +export interface CreateNodeOptions { + /** + * Filter or modify values while creating a node. + * + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#The_replacer_parameter + */ + replacer?: Replacer + /** + * Specify the collection type, e.g. `"!!omap"`. Note that this requires the + * corresponding tag to be available in this document's schema. + */ + tag?: string + /** + * Wrap plain values in `Scalar` objects. + * + * Default: `true` + */ + wrapScalars?: boolean +} + +export class Document extends Collection { + cstNode?: CST.Document + /** + * @param value - The initial value for the document, which will be wrapped + * in a Node container. + */ + constructor(value?: any, options?: Options) + constructor(value: any, replacer: null | Replacer, options?: Options) + tag: never + directivesEndMarker?: boolean + type: Type.DOCUMENT + /** + * Anchors associated with the document's nodes; + * also provides alias & merge node creators. + */ + anchors: Document.Anchors + /** The document contents. */ + contents: any + /** Errors encountered during parsing. */ + errors: YAMLError[] + /** + * The schema used with the document. Use `setSchema()` to change or + * initialise. + */ + schema?: Schema + /** + * Array of prefixes; each will have a string `handle` that + * starts and ends with `!` and a string `prefix` that the handle will be replaced by. + */ + tagPrefixes: Document.TagPrefix[] + /** + * The parsed version of the source document; + * if true-ish, stringified output will include a `%YAML` directive. + */ + version?: string + /** Warnings encountered during parsing. */ + warnings: YAMLWarning[] + + /** + * Convert any value into a `Node` using the current schema, recursively + * turning objects into collections. + */ + createNode( + value: any, + { replacer, tag, wrapScalars }?: CreateNodeOptions + ): Node + /** + * Convert a key and a value into a `Pair` using the current schema, + * recursively wrapping all values as `Scalar` or `Collection` nodes. + * + * @param options If `wrapScalars` is not `false`, wraps plain values in + * `Scalar` objects. + */ + createPair(key: any, value: any, options?: { wrapScalars?: boolean }): Pair + /** + * List the tags used in the document that are not in the default + * `tag:yaml.org,2002:` namespace. + */ + listNonDefaultTags(): string[] + /** Parse a CST into this document */ + parse(cst: CST.Document): this + /** + * When a document is created with `new YAML.Document()`, the schema object is + * not set as it may be influenced by parsed directives; call this with no + * arguments to set it manually, or with arguments to change the schema used + * by the document. + */ + setSchema( + id?: Options['version'] | Schema.Name, + customTags?: (Schema.TagId | Schema.Tag)[] + ): void + /** Set `handle` as a shorthand string for the `prefix` tag namespace. */ + setTagPrefix(handle: string, prefix: string): void + /** + * A plain JavaScript representation of the document `contents`. + * + * @param mapAsMap - Use Map rather than Object to represent mappings. + * Overrides values set in Document or global options. + * @param onAnchor - If defined, called with the resolved `value` and + * reference `count` for each anchor in the document. + * @param reviver - A function that may filter or modify the output JS value + */ + toJS(opt?: { + mapAsMap?: boolean + onAnchor?: (value: any, count: number) => void + reviver?: Reviver + }): any + /** + * A JSON representation of the document `contents`. + * + * @param arg Used by `JSON.stringify` to indicate the array index or property + * name. + */ + toJSON(arg?: string): any + /** A YAML representation of the document. */ + toString(): string +} + +export namespace Document { + interface Parsed extends Document { + contents: Node | null + /** The schema used with the document. */ + schema: Schema + } + + interface Anchors { + /** + * Create a new `Alias` node, adding the required anchor for `node`. + * If `name` is empty, a new anchor name will be generated. + */ + createAlias(node: Node, name?: string): Alias + /** + * Create a new `Merge` node with the given source nodes. + * Non-`Alias` sources will be automatically wrapped. + */ + createMergePair(...nodes: Node[]): Merge + /** The anchor name associated with `node`, if set. */ + getName(node: Node): undefined | string + /** List of all defined anchor names. */ + getNames(): string[] + /** The node associated with the anchor `name`, if set. */ + getNode(name: string): undefined | Node + /** + * Find an available anchor name with the given `prefix` and a + * numerical suffix. + */ + newName(prefix: string): string + /** + * Associate an anchor with `node`. If `name` is empty, a new name will be generated. + * To remove an anchor, use `setAnchor(null, name)`. + */ + setAnchor(node: Node | null, name?: string): void | string + } + + interface TagPrefix { + handle: string + prefix: string + } +} diff --git a/src/doc/Schema.d.ts b/src/doc/Schema.d.ts new file mode 100644 index 00000000..a707d958 --- /dev/null +++ b/src/doc/Schema.d.ts @@ -0,0 +1,168 @@ +import { Document } from './Document' +import { Node, Pair, Scalar, YAMLMap, YAMLSeq } from '../ast' +import { CST } from '../cst' + +export class Schema { + constructor(options: Schema.Options) + knownTags: { [key: string]: Schema.CustomTag } + merge: boolean + name: Schema.Name + sortMapEntries: ((a: Pair, b: Pair) => number) | null + tags: Schema.Tag[] +} + +export namespace Schema { + type Name = 'core' | 'failsafe' | 'json' | 'yaml-1.1' + + interface Options { + /** + * Array of additional tags to include in the schema, or a function that may + * modify the schema's base tag array. + */ + customTags?: (TagId | Tag)[] | ((tags: Tag[]) => Tag[]) + /** + * Enable support for `<<` merge keys. + * + * Default: `false` for YAML 1.2, `true` for earlier versions + */ + merge?: boolean + /** + * When using the `'core'` schema, support parsing values with these + * explicit YAML 1.1 tags: + * + * `!!binary`, `!!omap`, `!!pairs`, `!!set`, `!!timestamp`. + * + * Default `true` + */ + resolveKnownTags?: boolean + /** + * The base schema to use. + * + * Default: `"core"` for YAML 1.2, `"yaml-1.1"` for earlier versions + */ + schema?: Name + /** + * When stringifying, sort map entries. If `true`, sort by comparing key values with `<`. + * + * Default: `false` + */ + sortMapEntries?: boolean | ((a: Pair, b: Pair) => number) + /** + * @deprecated Use `customTags` instead. + */ + tags?: Options['customTags'] + } + + interface CreateNodeContext { + wrapScalars?: boolean + [key: string]: any + } + + interface StringifyContext { + forceBlockIndent?: boolean + implicitKey?: boolean + indent?: string + indentAtStart?: number + inFlow?: boolean + [key: string]: any + } + + type TagId = + | 'binary' + | 'bool' + | 'float' + | 'floatExp' + | 'floatNaN' + | 'floatTime' + | 'int' + | 'intHex' + | 'intOct' + | 'intTime' + | 'null' + | 'omap' + | 'pairs' + | 'set' + | 'timestamp' + + type Tag = CustomTag | DefaultTag + + interface BaseTag { + /** + * An optional factory function, used e.g. by collections when wrapping JS objects as AST nodes. + */ + createNode?: ( + schema: Schema, + value: any, + ctx: Schema.CreateNodeContext + ) => YAMLMap | YAMLSeq | Scalar + /** + * If a tag has multiple forms that should be parsed and/or stringified differently, use `format` to identify them. + */ + format?: string + /** + * Used by `YAML.createNode` to detect your data type, e.g. using `typeof` or + * `instanceof`. + */ + identify(value: any): boolean + /** + * The `Node` child class that implements this tag. Required for collections and tags that have overlapping JS representations. + */ + nodeClass?: new () => any + /** + * Used by some tags to configure their stringification, where applicable. + */ + options?: object + /** + * Optional function stringifying the AST node in the current context. If your + * data includes a suitable `.toString()` method, you can probably leave this + * undefined and use the default stringifier. + * + * @param item The node being stringified. + * @param ctx Contains the stringifying context variables. + * @param onComment Callback to signal that the stringifier includes the + * item's comment in its output. + * @param onChompKeep Callback to signal that the output uses a block scalar + * type with the `+` chomping indicator. + */ + /** + * Turns a value into an AST node. + * If returning a non-`Node` value, the output will be wrapped as a `Scalar`. + */ + resolve( + value: string | YAMLMap | YAMLSeq, + onError: (message: string) => void + ): Node | any + stringify?: ( + item: Node, + ctx: Schema.StringifyContext, + onComment?: () => void, + onChompKeep?: () => void + ) => string + /** + * The identifier for your data type, with which its stringified form will be + * prefixed. Should either be a !-prefixed local `!tag`, or a fully qualified + * `tag:domain,date:foo`. + */ + tag: string + } + + interface CustomTag extends BaseTag { + default?: false + } + + interface DefaultTag extends BaseTag { + /** + * If `true`, together with `test` allows for values to be stringified without + * an explicit tag. For most cases, it's unlikely that you'll actually want to + * use this, even if you first think you do. + */ + default: true + /** + * Together with `default` allows for values to be stringified without an + * explicit tag and detected using a regular expression. For most cases, it's + * unlikely that you'll actually want to use these, even if you first think + * you do. + */ + test: RegExp + } +} diff --git a/src/errors.d.ts b/src/errors.d.ts new file mode 100644 index 00000000..2c184e68 --- /dev/null +++ b/src/errors.d.ts @@ -0,0 +1,44 @@ +import { Type } from './constants' +import { CST } from './cst' + +interface LinePos { + line: number + col: number +} + +export class YAMLError extends Error { + name: + | 'YAMLReferenceError' + | 'YAMLSemanticError' + | 'YAMLSyntaxError' + | 'YAMLWarning' + message: string + source?: CST.Node + + nodeType?: Type + range?: CST.Range + linePos?: { start: LinePos; end: LinePos } + + /** + * Drops `source` and adds `nodeType`, `range` and `linePos`, as well as + * adding details to `message`. Run automatically for document errors if + * the `prettyErrors` option is set. + */ + makePretty(): void +} + +export class YAMLReferenceError extends YAMLError { + name: 'YAMLReferenceError' +} + +export class YAMLSemanticError extends YAMLError { + name: 'YAMLSemanticError' +} + +export class YAMLSyntaxError extends YAMLError { + name: 'YAMLSyntaxError' +} + +export class YAMLWarning extends YAMLError { + name: 'YAMLWarning' +} diff --git a/src/options.d.ts b/src/options.d.ts new file mode 100644 index 00000000..fe18e011 --- /dev/null +++ b/src/options.d.ts @@ -0,0 +1,192 @@ +import { Scalar } from './ast' +import { Schema } from './doc/Schema' + +/** + * `yaml` defines document-specific options in three places: as an argument of + * parse, create and stringify calls, in the values of `YAML.defaultOptions`, + * and in the version-dependent `YAML.Document.defaults` object. Values set in + * `YAML.defaultOptions` override version-dependent defaults, and argument + * options override both. + */ +export const defaultOptions: Options + +export interface Options extends Schema.Options { + /** + * Default prefix for anchors. + * + * Default: `'a'`, resulting in anchors `a1`, `a2`, etc. + */ + anchorPrefix?: string + /** + * The number of spaces to use when indenting code. + * + * Default: `2` + */ + indent?: number + /** + * Whether block sequences should be indented. + * + * Default: `true` + */ + indentSeq?: boolean + /** + * Include references in the AST to each node's corresponding CST node. + * + * Default: `false` + */ + keepCstNodes?: boolean + /** + * Store the original node type when parsing documents. + * + * Default: `true` + */ + keepNodeTypes?: boolean + /** + * Keep `undefined` object values when creating mappings and return a Scalar + * node when calling `YAML.stringify(undefined)`, rather than `undefined`. + * + * Default: `false` + */ + keepUndefined?: boolean + /** + * When outputting JS, use Map rather than Object to represent mappings. + * + * Default: `false` + */ + mapAsMap?: boolean + /** + * Prevent exponential entity expansion attacks by limiting data aliasing count; + * set to `-1` to disable checks; `0` disallows all alias nodes. + * + * Default: `100` + */ + maxAliasCount?: number + /** + * Include line position & node type directly in errors; drop their verbose source and context. + * + * Default: `true` + */ + prettyErrors?: boolean + /** + * When stringifying, require keys to be scalars and to use implicit rather than explicit notation. + * + * Default: `false` + */ + simpleKeys?: boolean + /** + * The YAML version used by documents without a `%YAML` directive. + * + * Default: `"1.2"` + */ + version?: '1.0' | '1.1' | '1.2' +} + +/** + * Some customization options are availabe to control the parsing and + * stringification of scalars. Note that these values are used by all documents. + */ +export const scalarOptions: { + binary: scalarOptions.Binary + bool: scalarOptions.Bool + int: scalarOptions.Int + null: scalarOptions.Null + str: scalarOptions.Str +} +export namespace scalarOptions { + interface Binary { + /** + * The type of string literal used to stringify `!!binary` values. + * + * Default: `'BLOCK_LITERAL'` + */ + defaultType: Scalar.Type + /** + * Maximum line width for `!!binary`. + * + * Default: `76` + */ + lineWidth: number + } + + interface Bool { + /** + * String representation for `true`. With the core schema, use `'true' | 'True' | 'TRUE'`. + * + * Default: `'true'` + */ + trueStr: string + /** + * String representation for `false`. With the core schema, use `'false' | 'False' | 'FALSE'`. + * + * Default: `'false'` + */ + falseStr: string + } + + interface Int { + /** + * Whether integers should be parsed into BigInt values. + * + * Default: `false` + */ + asBigInt: boolean + } + + interface Null { + /** + * String representation for `null`. With the core schema, use `'null' | 'Null' | 'NULL' | '~' | ''`. + * + * Default: `'null'` + */ + nullStr: string + } + + interface Str { + /** + * The default type of string literal used to stringify values in general + * + * Default: `'PLAIN'` + */ + defaultType: Scalar.Type + /** + * The default type of string literal used to stringify implicit key values + * + * Default: `'PLAIN'` + */ + defaultKeyType: Scalar.Type + /** + * Use 'single quote' rather than "double quote" by default + * + * Default: `false` + */ + defaultQuoteSingle: boolean + doubleQuoted: { + /** + * Whether to restrict double-quoted strings to use JSON-compatible syntax. + * + * Default: `false` + */ + jsonEncoding: boolean + /** + * Minimum length to use multiple lines to represent the value. + * + * Default: `40` + */ + minMultiLineLength: number + } + fold: { + /** + * Maximum line width (set to `0` to disable folding). + * + * Default: `80` + */ + lineWidth: number + /** + * Minimum width for highly-indented content. + * + * Default: `20` + */ + minContentWidth: number + } + } +} From b270dfbe22c2323d7593a0bb2ed0758b01074aa6 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 01:22:22 +0300 Subject: [PATCH 09/89] Refactor & fix type declarations --- src/ast/index.d.ts | 6 +++++- src/doc/Document.d.ts | 7 ++++++- src/doc/Schema.d.ts | 43 +++++++++++++++++-------------------------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/ast/index.d.ts b/src/ast/index.d.ts index 46ca2a51..f203744c 100644 --- a/src/ast/index.d.ts +++ b/src/ast/index.d.ts @@ -30,8 +30,9 @@ export class Scalar extends Node { /** * By default (undefined), numbers use decimal notation. * The YAML 1.2 core schema only supports 'HEX' and 'OCT'. + * The YAML 1.1 schema also supports 'BIN' and 'TIME' */ - format?: 'BIN' | 'HEX' | 'OCT' | 'TIME' + format?: string value: any toJSON(arg?: any, ctx?: AST.NodeToJsonContext): any toString(): string @@ -46,6 +47,7 @@ export namespace Scalar { } export class Alias extends Node { + constructor(source: Node) type: Type.ALIAS source: Node cstNode?: CST.Alias @@ -88,6 +90,8 @@ export class Collection extends Node { items: any[] schema?: Schema + constructor(schema?: Schema) + /** * Adds a value to the collection. For `!!map` and `!!omap` the value must * be a Pair instance or a `{ key, value }` object, which may not have a key diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts index d90f0321..d3ca208a 100644 --- a/src/doc/Document.d.ts +++ b/src/doc/Document.d.ts @@ -1,4 +1,4 @@ -import { Alias, Collection, Merge, Pair } from '../ast' +import { Alias, Collection, Merge, Node, Pair } from '../ast' import { Type } from '../constants' import { CST } from '../cst' import { YAMLError, YAMLWarning } from '../errors' @@ -134,6 +134,11 @@ export namespace Document { } interface Anchors { + /** @private */ + _cstAliases: any[] + /** @private */ + map: Record + /** * Create a new `Alias` node, adding the required anchor for `node`. * If `name` is empty, a new anchor name will be generated. diff --git a/src/doc/Schema.d.ts b/src/doc/Schema.d.ts index a707d958..d30a3e24 100644 --- a/src/doc/Schema.d.ts +++ b/src/doc/Schema.d.ts @@ -4,7 +4,7 @@ import { CST } from '../cst' export class Schema { constructor(options: Schema.Options) - knownTags: { [key: string]: Schema.CustomTag } + knownTags: { [key: string]: Schema.Tag } merge: boolean name: Schema.Name sortMapEntries: ((a: Pair, b: Pair) => number) | null @@ -84,9 +84,7 @@ export namespace Schema { | 'set' | 'timestamp' - type Tag = CustomTag | DefaultTag - - interface BaseTag { + interface Tag { /** * An optional factory function, used e.g. by collections when wrapping JS objects as AST nodes. */ @@ -95,6 +93,12 @@ export namespace Schema { value: any, ctx: Schema.CreateNodeContext ) => YAMLMap | YAMLSeq | Scalar + /** + * If `true`, together with `test` allows for values to be stringified without + * an explicit tag. For most cases, it's unlikely that you'll actually want to + * use this, even if you first think you do. + */ + default: boolean /** * If a tag has multiple forms that should be parsed and/or stringified differently, use `format` to identify them. */ @@ -112,6 +116,14 @@ export namespace Schema { * Used by some tags to configure their stringification, where applicable. */ options?: object + /** + * Turns a value into an AST node. + * If returning a non-`Node` value, the output will be wrapped as a `Scalar`. + */ + resolve( + value: string | YAMLMap | YAMLSeq, + onError: (message: string) => void + ): Node | any /** * Optional function stringifying the AST node in the current context. If your * data includes a suitable `.toString()` method, you can probably leave this @@ -124,14 +136,6 @@ export namespace Schema { * @param onChompKeep Callback to signal that the output uses a block scalar * type with the `+` chomping indicator. */ - /** - * Turns a value into an AST node. - * If returning a non-`Node` value, the output will be wrapped as a `Scalar`. - */ - resolve( - value: string | YAMLMap | YAMLSeq, - onError: (message: string) => void - ): Node | any stringify?: ( item: Node, ctx: Schema.StringifyContext, @@ -144,25 +148,12 @@ export namespace Schema { * `tag:domain,date:foo`. */ tag: string - } - - interface CustomTag extends BaseTag { - default?: false - } - - interface DefaultTag extends BaseTag { - /** - * If `true`, together with `test` allows for values to be stringified without - * an explicit tag. For most cases, it's unlikely that you'll actually want to - * use this, even if you first think you do. - */ - default: true /** * Together with `default` allows for values to be stringified without an * explicit tag and detected using a regular expression. For most cases, it's * unlikely that you'll actually want to use these, even if you first think * you do. */ - test: RegExp + test?: RegExp } } From fdd097131aa83e898fe0f08624647d081f7ae56f Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 01:24:31 +0300 Subject: [PATCH 10/89] Add initial dev-only build for mixed JS+TS sources For now with separate build steps: npx rollup -c rollup.dev-config.js npx tsc --- .gitignore | 1 + rollup.dev-config.js | 7 +++++++ tsconfig.json | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 rollup.dev-config.js diff --git a/.gitignore b/.gitignore index 56921844..1a62b656 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .* coverage/ dist/ +lib/ node_modules/ diff --git a/rollup.dev-config.js b/rollup.dev-config.js new file mode 100644 index 00000000..dd2fd13b --- /dev/null +++ b/rollup.dev-config.js @@ -0,0 +1,7 @@ +import babel from '@rollup/plugin-babel' + +export default { + input: ['src/index.js', 'src/test-events.js', 'src/types.js', 'src/util.js', 'src/ast/index.js'], + output: { dir: 'lib', format: 'cjs', esModule: false, preserveModules: true }, + plugins: [babel()] +} diff --git a/tsconfig.json b/tsconfig.json index 67eca3be..7692848d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,7 @@ "moduleResolution": "node", "noUnusedLocals": true, "noUnusedParameters": true, - "outDir": "lib/", + "outDir": "lib/stream/", "strict": true, "target": "ES2017" }, From 6fc22cba435297272036e435ee7e87ac85858179 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 01:26:16 +0300 Subject: [PATCH 11/89] Add StreamDirectives --- src/stream/stream-directives.ts | 91 +++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/stream/stream-directives.ts diff --git a/src/stream/stream-directives.ts b/src/stream/stream-directives.ts new file mode 100644 index 00000000..1f089c35 --- /dev/null +++ b/src/stream/stream-directives.ts @@ -0,0 +1,91 @@ +export class StreamDirectives { + tags: Record = { '!!': 'tag:yaml.org,2002:' } + yaml: { version: '1.1' | '1.2' } = { version: '1.2' } + + static from(src: StreamDirectives) { + const res = new StreamDirectives() + Object.assign(res.tags, src.tags) + Object.assign(res.yaml, src.yaml) + return res + } + + /** + * @param onError - May be called even if the action was successful + * @returns `true` on success + */ + add(line: string, onError: (offset: number, message: string) => void) { + const parts = line.trim().split(/[ \t]+/) + const name = parts.shift() + switch (name) { + case '%TAG': { + if (parts.length !== 2) { + onError(0, '%TAG directive should contain exactly two parts') + if (parts.length < 2) return false + } + const [handle, prefix] = parts + this.tags[handle] = prefix + return true + } + case '%YAML': { + if (parts.length < 1) { + onError(0, '%YAML directive should contain exactly one part') + return false + } + const [version] = parts + if (version === '1.1' || version === '1.2') { + this.yaml.version = version + return true + } else { + onError(6, `Unsupported YAML version ${version}`) + return false + } + } + default: + onError(0, `Unknown directive ${name}`) + return false + } + } + + /** + * Resolves a tag, matching handles to those defined in %TAG directives. + * + * @returns Resolved tag, which may also be the non-specific tag `'!'` or a + * `'!local'` tag, or `null` if unresolvable. + */ + tagName(source: string, onError: (offset: number, message: string) => void) { + if (source === '!') return '!' // non-specific tag + + if (source[0] !== '!') { + onError(0, `Not a valid tag: ${source}`) + return null + } + + if (source[1] === '<') { + const verbatim = source.slice(2, -1) + if (verbatim === '!' || verbatim === '!!') { + onError(0, `Verbatim tags aren't resolved, so ${source} is invalid.`) + return null + } + if (source[source.length - 1] !== '>') + onError(source.length - 1, 'Verbatim tags must end with a >') + return verbatim + } + + const [, handle, suffix] = source.match(/^(.*!)([^!]*)$/) as string[] + const prefix = this.tags[handle] + if (prefix) return prefix + suffix + if (handle === '!') return source // local tag + + onError(0, `Could not resolve tag: ${source}`) + return null + } + + toString(includeVersion: boolean) { + let res = includeVersion ? `%YAML ${this.yaml.version}\n` : '' + for (const [handle, prefix] of Object.entries(this.tags)) { + if (handle !== '!!' || prefix !== 'tag:yaml.org,2002:') + res += `%TAG ${handle} ${prefix}\n` + } + return res + } +} From 9d0970957a4777523c37ceb5f1c086c52556e444 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 01:29:39 +0300 Subject: [PATCH 12/89] Add composeScalar() + scalar value folding --- src/stream/block-scalar-value.ts | 166 +++++++++++++++++++++++++++++++ src/stream/compose-scalar.ts | 82 +++++++++++++++ src/stream/flow-scalar-value.ts | 165 ++++++++++++++++++++++++++++++ src/stream/test.ts | 2 +- 4 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 src/stream/block-scalar-value.ts create mode 100644 src/stream/compose-scalar.ts create mode 100644 src/stream/flow-scalar-value.ts diff --git a/src/stream/block-scalar-value.ts b/src/stream/block-scalar-value.ts new file mode 100644 index 00000000..ced4cc2a --- /dev/null +++ b/src/stream/block-scalar-value.ts @@ -0,0 +1,166 @@ +import { Type } from '../constants.js' +import { BlockScalar, Token } from './parser.js' + +export function blockScalarValue( + scalar: BlockScalar, + onError: (offset: number, message: string) => void, + onType: (type: Type.BLOCK_LITERAL | Type.BLOCK_FOLDED) => void +) { + const header = parseBlockScalarHeader(scalar.props, onError) + if (!header || !scalar.source) return '' + const lines = splitLines(scalar.source) + + // determine the end of content & start of chomping + let chompStart = lines.length + for (let i = lines.length - 1; i >= 0; --i) + if (lines[i][1] === '') chompStart = i + else break + + // shortcut for empty contents + if (chompStart === 0) { + if (header.chomp === '+') + return lines.map(line => line[0]).join('\n') + '\n' + else return header.chomp === '-' ? '' : '\n' + } + + // find the indentation level to trim from start + let trimIndent = scalar.indent + header.indent + let offset = header.length + let contentStart = 0 + for (let i = 0; i < chompStart; ++i) { + const [indent, content] = lines[i] + if (content === '') { + if (header.indent === 0 && indent.length > trimIndent) + trimIndent = indent.length + } else { + if (indent.length < trimIndent) { + const message = + 'Block scalars with more-indented leading empty lines must use an explicit indentation indicator' + onError(offset + indent.length, message) + } + trimIndent = indent.length + contentStart = i + break + } + offset += indent.length + content.length + 1 + } + + let res = '' + let sep = '' + let prevMoreIndented = false + + // leading whitespace is kept intact + for (let i = 0; i < contentStart; ++i) + res += lines[i][0].slice(trimIndent) + '\n' + + const folded = header.mode === '>' + onType(folded ? Type.BLOCK_FOLDED : Type.BLOCK_LITERAL) + for (let i = contentStart; i < chompStart; ++i) { + let [indent, content] = lines[i] + offset += indent.length + content.length + 1 + const crlf = content[content.length - 1] === '\r' + if (crlf) content = content.slice(0, -1) + + if (indent.length < trimIndent) { + if (content === '') { + // empty line + if (sep === '\n') res += '\n' + else sep = '\n' + continue + } else { + const src = header.indent + ? 'explicit indentation indicator' + : 'first line' + const message = `Block scalar lines must not be less indented than their ${src}` + onError(offset - content.length - (crlf ? 2 : 1), message) + indent = '' + } + } + + if (folded) { + if (!indent || indent.length === trimIndent) { + res += sep + content + sep = ' ' + prevMoreIndented = false + } else { + // more-indented content within a folded block + if (sep === ' ') sep = '\n' + else if (!prevMoreIndented && sep === '\n') sep = '\n\n' + res += sep + indent.slice(trimIndent) + content + sep = '\n' + prevMoreIndented = true + } + } else { + // literal + res += sep + indent.slice(trimIndent) + content + sep = '\n' + } + } + + switch (header.chomp) { + case '-': + return res + case '+': + for (let i = chompStart; i < lines.length; ++i) + res += '\n' + lines[i][0].slice(trimIndent) + return res[res.length - 1] === '\n' ? res : res + '\n' + default: + return res + '\n' + } +} + +function parseBlockScalarHeader( + props: Token[], + onError: (offset: number, message: string) => void +) { + if (props[0].type !== 'block-scalar-header') { + onError(0, 'Block scalar header not found') + return null + } + const { source } = props[0] + const mode = source[0] as '>' | '|' + let indent = 0 + let chomp: '' | '-' | '+' = '' + let error = -1 + for (let i = 1; i < source.length; ++i) { + const ch = source[i] + if (!chomp && (ch === '-' || ch === '+')) chomp = ch + else { + const n = Number(ch) + if (!indent && n) indent = n + else if (error === -1) error = i + } + } + if (error !== -1) + onError(error, `Block scalar header includes extra characters: ${source}`) + let length = source.length + for (let i = 1; i < props.length; ++i) { + const token = props[i] + switch (token.type) { + case 'space': + case 'comment': + case 'newline': + length += token.source.length + break + default: { + const message = `Unexpected token in block scalar header: ${token.type}` + onError(length, message) + const ts = (token as any).source || '' + if (typeof ts === 'string') length += ts.length + } + } + } + return { mode, indent, chomp, length } +} + +/** @returns Array of lines split up as `[indent, content]` */ +function splitLines(source: string) { + const split = source.split(/\n( *)/) + const first = split[0] + const m = first.match(/^( *)/) + const line0: [string, string] = + m && m[1] ? [m[1], first.slice(m[1].length)] : ['', first] + const lines = [line0] + for (let i = 1; i < split.length; i += 2) lines.push([split[i], split[i + 1]]) + return lines +} diff --git a/src/stream/compose-scalar.ts b/src/stream/compose-scalar.ts new file mode 100644 index 00000000..80a8a394 --- /dev/null +++ b/src/stream/compose-scalar.ts @@ -0,0 +1,82 @@ +import { Scalar } from '../ast' +import { Type } from '../constants' +import { Schema } from '../doc/Schema' +import { blockScalarValue } from './block-scalar-value' +import { flowScalarValue } from './flow-scalar-value' +import { BlockScalar, SourceToken } from './parser' + +export function composeScalar( + schema: Schema, + tagName: string | null, + token: SourceToken | BlockScalar, + onError: (offset: number, message: string, warning?: boolean) => void +) { + let type: + | Type.BLOCK_FOLDED + | Type.BLOCK_LITERAL + | Type.PLAIN + | Type.QUOTE_DOUBLE + | Type.QUOTE_SINGLE + | null = null + const onType = (t: typeof type) => { + type = t + } + const value = + token.type === 'block-scalar' + ? blockScalarValue(token, onError, onType) + : flowScalarValue(token, onError, onType) + const tag = + findScalarTagByName(schema, value, tagName, onError) || + findScalarTagByTest(schema, value, token.type === 'scalar') || + findScalarTagByName(schema, value, '!', onError) + + let scalar: Scalar + try { + const res = tag ? tag.resolve(value, message => onError(0, message)) : value + scalar = res instanceof Scalar ? res : new Scalar(res) + } catch (error) { + onError(0, error.message) + scalar = new Scalar(value) + } + if (type) scalar.type = type + if (tagName) scalar.tag = tagName + if (tag && tag.format) scalar.format = tag.format + return scalar +} + +function findScalarTagByName( + schema: Schema, + value: string, + tagName: string | null, + onError: (offset: number, message: string, warning?: boolean) => void +) { + if (!tagName) return null + if (tagName === '!') tagName = 'tag:yaml.org,2002:str' // non-specific tag + const matchWithTest: Schema.Tag[] = [] + for (const tag of schema.tags) { + if (tag.tag === tagName) { + if (tag.default && tag.test) matchWithTest.push(tag) + else return tag + } + } + for (const tag of matchWithTest) + if (tag.test && tag.test.test(value)) return tag + const kt = schema.knownTags[tagName] + if (kt) { + // Ensure that the known tag is available for stringifying, + // but does not get used by default. + schema.tags.push(Object.assign({}, kt, { default: false, test: undefined })) + return kt + } + onError(0, `Unresolved tag: ${tagName}`, tagName !== 'tag:yaml.org,2002:str') + return null +} + +function findScalarTagByTest(schema: Schema, value: string, apply: boolean) { + if (apply) { + for (const tag of schema.tags) { + if (tag.default && tag.test && tag.test.test(value)) return tag + } + } + return null +} diff --git a/src/stream/flow-scalar-value.ts b/src/stream/flow-scalar-value.ts new file mode 100644 index 00000000..4cabe394 --- /dev/null +++ b/src/stream/flow-scalar-value.ts @@ -0,0 +1,165 @@ +import { Type } from '../constants' +import { SourceToken } from './parser' + +export function flowScalarValue( + { type, source }: SourceToken, + onError: (offset: number, message: string) => void, + onType: (type: Type.PLAIN | Type.QUOTE_DOUBLE | Type.QUOTE_SINGLE) => void +) { + switch (type) { + case 'scalar': + onType(Type.PLAIN) + return plainValue(source, onError) + + case 'single-quoted-scalar': + onType(Type.QUOTE_SINGLE) + return singleQuotedValue(source, onError) + + case 'double-quoted-scalar': + onType(Type.QUOTE_DOUBLE) + return doubleQuotedValue(source, onError) + } + onError(0, `Expected a flow scalar value, but found: ${type}`) + return '' +} + +function plainValue( + source: string, + onError: (offset: number, message: string) => void +) { + switch (source[0]) { + case '\t': + onError(0, 'Plain value cannot start with a tab character') + break + case '@': + case '`': { + const message = `Plain value cannot start with reserved character ${source[0]}` + onError(0, message) + break + } + } + return foldLines(source.trim()) +} + +function singleQuotedValue( + source: string, + onError: (offset: number, message: string) => void +) { + if (source[source.length - 1] !== "'") + onError(source.length, "Missing closing 'quote") + return foldLines(source.slice(1, -1)).replace(/''/g, "'") +} + +function foldLines(source: string) { + const lines = source.split(/[ \t]*\n[ \t]*/) + let res = '' + let sep = '' + for (let i = 0; i < lines.length; ++i) { + const line = lines[i] + if (line === '') { + if (sep === '\n') res += sep + else sep = '\n' + } else { + res += sep + line + sep = ' ' + } + } + return res +} + +function doubleQuotedValue( + source: string, + onError: (offset: number, message: string) => void +) { + let res = '' + for (let i = 1; i < source.length - 1; ++i) { + const ch = source[i] + if (ch === '\n') { + const { fold, offset } = foldNewline(source, i) + res += fold + i = offset + } else if (ch === '\\') { + let next = source[++i] + const cc = escapeCodes[next] + if (cc) res += cc + else if (next === '\n') { + // skip escaped newlines, but still trim the following line + next = source[i + 1] + while (next === ' ' || next === '\t') next = source[++i + 1] + } else if (next === 'x' || next === 'u' || next === 'U') { + const length = { x: 2, u: 4, U: 8 }[next] + res += parseCharCode(source, i + 1, length, onError) + i += length + } else { + const raw = source.substr(i - 1, 2) + onError(i - 1, `Invalid escape sequence ${raw}`) + res += raw + } + } else if (ch === ' ' || ch === '\t') { + // trim trailing whitespace + const wsStart = i + let next = source[i + 1] + while (next === ' ' || next === '\t') next = source[++i + 1] + if (next !== '\n') res += i > wsStart ? source.slice(wsStart, i + 1) : ch + } else { + res += ch + } + } + if (source[source.length - 1] !== '"') + onError(source.length, 'Missing closing "quote') + return res +} + +/** + * Fold a single newline into a space, multiple newlines to N - 1 newlines. + * Presumes `source[offset] === '\n'` + */ +function foldNewline(source: string, offset: number) { + let fold = '' + let ch = source[offset + 1] + while (ch === ' ' || ch === '\t' || ch === '\n') { + if (ch === '\n') fold += '\n' + offset += 1 + ch = source[offset + 1] + } + if (!fold) fold = ' ' + return { fold, offset } +} + +const escapeCodes: Record = { + '0': '\0', // null character + a: '\x07', // bell character + b: '\b', // backspace + e: '\x1b', // escape character + f: '\f', // form feed + n: '\n', // line feed + r: '\r', // carriage return + t: '\t', // horizontal tab + v: '\v', // vertical tab + N: '\u0085', // Unicode next line + _: '\u00a0', // Unicode non-breaking space + L: '\u2028', // Unicode line separator + P: '\u2029', // Unicode paragraph separator + ' ': ' ', + '"': '"', + '/': '/', + '\\': '\\', + '\t': '\t' +} + +function parseCharCode( + source: string, + offset: number, + length: number, + onError: (offset: number, message: string) => void +) { + const cc = source.substr(offset, length) + const ok = cc.length === length && /^[0-9a-fA-F]+$/.test(cc) + const code = ok ? parseInt(cc, 16) : NaN + if (isNaN(code)) { + const raw = source.substr(offset - 2, length + 2) + onError(offset - 2, `Invalid escape sequence ${raw}`) + return raw + } + return String.fromCodePoint(code) +} diff --git a/src/stream/test.ts b/src/stream/test.ts index 74ad512f..0d226b56 100644 --- a/src/stream/test.ts +++ b/src/stream/test.ts @@ -7,7 +7,7 @@ export function stream(source: string) { ps.end() } -export function sync(source: string) { +export function test(source: string) { const parser = new Parser(t => console.dir(t, { depth: null })) parser.parse(source) } From c409eeef0968a259bc51ef2de11c6d5df1f91ebf Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 09:58:39 +0300 Subject: [PATCH 13/89] Split src/stream/ into src/{parse,compose}/ --- src/{stream => compose}/block-scalar-value.ts | 2 +- src/{stream => compose}/compose-scalar.ts | 34 +++++++++++++++---- src/{stream => compose}/flow-scalar-value.ts | 4 +-- src/{stream => compose}/stream-directives.ts | 0 src/{stream => parse}/lexer.ts | 0 src/{stream => parse}/parse-stream.ts | 0 src/{stream => parse}/parser.ts | 4 +-- src/{stream => parse}/test.ts | 0 src/{stream => parse}/token-type.ts | 0 tsconfig.json | 2 +- 10 files changed, 34 insertions(+), 12 deletions(-) rename src/{stream => compose}/block-scalar-value.ts (98%) rename src/{stream => compose}/compose-scalar.ts (72%) rename src/{stream => compose}/flow-scalar-value.ts (97%) rename src/{stream => compose}/stream-directives.ts (100%) rename src/{stream => parse}/lexer.ts (100%) rename src/{stream => parse}/parse-stream.ts (100%) rename src/{stream => parse}/parser.ts (99%) rename src/{stream => parse}/test.ts (100%) rename src/{stream => parse}/token-type.ts (100%) diff --git a/src/stream/block-scalar-value.ts b/src/compose/block-scalar-value.ts similarity index 98% rename from src/stream/block-scalar-value.ts rename to src/compose/block-scalar-value.ts index ced4cc2a..cea63ff8 100644 --- a/src/stream/block-scalar-value.ts +++ b/src/compose/block-scalar-value.ts @@ -1,5 +1,5 @@ import { Type } from '../constants.js' -import { BlockScalar, Token } from './parser.js' +import type { BlockScalar, Token } from '../parse/parser.js' export function blockScalarValue( scalar: BlockScalar, diff --git a/src/stream/compose-scalar.ts b/src/compose/compose-scalar.ts similarity index 72% rename from src/stream/compose-scalar.ts rename to src/compose/compose-scalar.ts index 80a8a394..1efc5317 100644 --- a/src/stream/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -1,9 +1,9 @@ -import { Scalar } from '../ast' -import { Type } from '../constants' -import { Schema } from '../doc/Schema' -import { blockScalarValue } from './block-scalar-value' -import { flowScalarValue } from './flow-scalar-value' -import { BlockScalar, SourceToken } from './parser' +import { Scalar } from '../ast/index.js' +import type { Type } from '../constants.js' +import type { Schema } from '../doc/Schema.js' +import type { BlockScalar, SourceToken, Token } from '../parse/parser.js' +import { blockScalarValue } from './block-scalar-value.js' +import { flowScalarValue } from './flow-scalar-value.js' export function composeScalar( schema: Schema, @@ -41,6 +41,12 @@ export function composeScalar( if (type) scalar.type = type if (tagName) scalar.tag = tagName if (tag && tag.format) scalar.format = tag.format + + // attach comments + const cs = token.type === 'block-scalar' ? token.props : token.end + const cv = cs && commentValue(cs) + if (cv) scalar.comment = cv + return scalar } @@ -80,3 +86,19 @@ function findScalarTagByTest(schema: Schema, value: string, apply: boolean) { } return null } + +function commentValue(tokens: Token[]) { + let comment = '' + let hasComment = false + let sep = '' + for (const c of tokens) { + if (c.type === 'comment') { + const cb = c.source.substring(1) + if (!hasComment) comment = cb + else comment += sep + cb + hasComment = true + sep = '' + } else if (hasComment && c.type === 'newline') sep += c.source + } + return comment +} diff --git a/src/stream/flow-scalar-value.ts b/src/compose/flow-scalar-value.ts similarity index 97% rename from src/stream/flow-scalar-value.ts rename to src/compose/flow-scalar-value.ts index 4cabe394..7eac0978 100644 --- a/src/stream/flow-scalar-value.ts +++ b/src/compose/flow-scalar-value.ts @@ -1,5 +1,5 @@ -import { Type } from '../constants' -import { SourceToken } from './parser' +import { Type } from '../constants.js' +import type { SourceToken } from '../parse/parser.js' export function flowScalarValue( { type, source }: SourceToken, diff --git a/src/stream/stream-directives.ts b/src/compose/stream-directives.ts similarity index 100% rename from src/stream/stream-directives.ts rename to src/compose/stream-directives.ts diff --git a/src/stream/lexer.ts b/src/parse/lexer.ts similarity index 100% rename from src/stream/lexer.ts rename to src/parse/lexer.ts diff --git a/src/stream/parse-stream.ts b/src/parse/parse-stream.ts similarity index 100% rename from src/stream/parse-stream.ts rename to src/parse/parse-stream.ts diff --git a/src/stream/parser.ts b/src/parse/parser.ts similarity index 99% rename from src/stream/parser.ts rename to src/parse/parser.ts index 94fdc966..1237af4f 100644 --- a/src/stream/parser.ts +++ b/src/parse/parser.ts @@ -1,5 +1,5 @@ -import { Lexer } from './lexer' -import { SourceTokenType, prettyToken, tokenType } from './token-type' +import { Lexer } from './lexer.js' +import { SourceTokenType, prettyToken, tokenType } from './token-type.js' export interface SourceToken { type: SourceTokenType diff --git a/src/stream/test.ts b/src/parse/test.ts similarity index 100% rename from src/stream/test.ts rename to src/parse/test.ts diff --git a/src/stream/token-type.ts b/src/parse/token-type.ts similarity index 100% rename from src/stream/token-type.ts rename to src/parse/token-type.ts diff --git a/tsconfig.json b/tsconfig.json index 7692848d..67eca3be 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,7 @@ "moduleResolution": "node", "noUnusedLocals": true, "noUnusedParameters": true, - "outDir": "lib/stream/", + "outDir": "lib/", "strict": true, "target": "ES2017" }, From b59e9301950ce6c5138d72943b69ad4e1a0df785 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 13:35:10 +0300 Subject: [PATCH 14/89] Add optional onNewLine(offset) callback to parser --- src/parse/parser.ts | 41 +++++++++++++++++++++++++++++++++++------ src/parse/test.ts | 7 ++++++- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 1237af4f..18a3e6a4 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -45,7 +45,7 @@ export interface BlockMap { export interface BlockSequence { type: 'block-seq' indent: number - items: Array<{ start: Token[]; value?: Token }> + items: Array<{ start: SourceToken[]; value?: Token }> } export interface FlowCollection { @@ -69,6 +69,7 @@ export type Token = /** A YAML concrete syntax tree parser */ export class Parser { push: (token: Token) => void + onNewLine?: (offset: number) => void lexer = new Lexer(ts => this.token(ts)) @@ -81,6 +82,8 @@ export class Parser { /** Current indentation level */ indent = 0 + offset = 0 + /** Top indicates the bode that's currently being built */ stack: Token[] = [] @@ -92,21 +95,27 @@ export class Parser { /** * @param push - Called separately with each parsed token + * @param onNewLine - If defined, called separately with the start position of each new line * @public */ - constructor(push: (token: Token) => void) { + constructor( + push: (token: Token) => void, + onNewLine?: (offset: number) => void + ) { this.push = push + this.onNewLine = onNewLine } /** - * Parse `source` as YAML, calling the constructor's callback once each - * directive, document and other structure is completely parsed. If `incomplete`, - * a part of the last line may be left as a buffer for the next call. + * Parse `source` as a YAML stream, calling `push` with each + * directive, document and other structure as it is completely parsed. + * If `incomplete`, a part of the last line may be left as a buffer for the next call. * * Errors are not thrown, but pushed out as `{ type: 'error', message }` tokens. * @public */ parse(source: string, incomplete = false) { + if (this.onNewLine && this.offset === 0) this.onNewLine(0) this.lexer.lex(source, incomplete) if (!incomplete) while (this.stack.length > 0) this.pop() } @@ -119,12 +128,15 @@ export class Parser { if (this.atScalar) { this.atScalar = false this.step() + this.offset += source.length return } + const type = tokenType(source) if (!type) { const message = `Not a YAML token: ${source}` this.pop({ type: 'error', source, message }) + this.offset += source.length } else if (type === 'scalar') { this.atNewLine = false this.atScalar = true @@ -136,6 +148,7 @@ export class Parser { case 'newline': this.atNewLine = true this.indent = 0 + if (this.onNewLine) this.onNewLine(this.offset + source.length) break case 'space': case 'seq-item-ind': @@ -146,6 +159,7 @@ export class Parser { default: this.atNewLine = false } + this.offset += source.length } } @@ -299,6 +313,13 @@ export class Parser { // block-scalar source includes trailing newline this.atNewLine = true this.indent = 0 + if (this.onNewLine) { + let nl = this.source.indexOf('\n') + 1 + while (nl !== 0) { + this.onNewLine(this.offset + nl) + nl = this.source.indexOf('\n', nl) + 1 + } + } this.pop() break default: @@ -389,7 +410,8 @@ export class Parser { return case 'seq-item-ind': if (this.indent !== seq.indent) break - if (it.value) seq.items.push({ start: [this.sourceToken] }) + if (it.value || it.start.some(tok => tok.type === 'seq-item-ind')) + seq.items.push({ start: [this.sourceToken] }) else it.start.push(this.sourceToken) return } @@ -437,6 +459,13 @@ export class Parser { case 'scalar': case 'single-quoted-scalar': case 'double-quoted-scalar': + if (this.onNewLine) { + let nl = this.source.indexOf('\n') + 1 + while (nl !== 0) { + this.onNewLine(this.offset + nl) + nl = this.source.indexOf('\n', nl) + 1 + } + } return st case 'block-scalar-header': return { diff --git a/src/parse/test.ts b/src/parse/test.ts index 0d226b56..62034675 100644 --- a/src/parse/test.ts +++ b/src/parse/test.ts @@ -8,6 +8,11 @@ export function stream(source: string) { } export function test(source: string) { - const parser = new Parser(t => console.dir(t, { depth: null })) + const lines: number[] = [] + const parser = new Parser( + t => console.dir(t, { depth: null }), + n => lines.push(n) + ) parser.parse(source) + console.log({ lines }) } From 3041d04499f530bc78b0258c961ae5a978bb4c3d Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 14:29:07 +0300 Subject: [PATCH 15/89] lexer: Require more indent in top-level mapping values --- src/parse/lexer.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index e3fcf4b9..868d7e1b 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -441,6 +441,8 @@ export class Lexer { if (this.indentMore) { this.indent += this.indentMore.length this.indentMore = '' + } else { + this.indentMore = ' ' } return this.pushCount(1) + this.pushSpaces() + this.pushIndicators() } From e4d2de3826d91c9bebf05becfed6dd293e394ed1 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 14:32:35 +0300 Subject: [PATCH 16/89] Add starting offset to all block & flow tokens --- src/compose/compose-scalar.ts | 4 +- src/compose/flow-scalar-value.ts | 4 +- src/parse/parser.ts | 107 ++++++++++++++++++++++--------- 3 files changed, 80 insertions(+), 35 deletions(-) diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index 1efc5317..d53a994d 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -1,14 +1,14 @@ import { Scalar } from '../ast/index.js' import type { Type } from '../constants.js' import type { Schema } from '../doc/Schema.js' -import type { BlockScalar, SourceToken, Token } from '../parse/parser.js' +import type { BlockScalar, FlowScalar, Token } from '../parse/parser.js' import { blockScalarValue } from './block-scalar-value.js' import { flowScalarValue } from './flow-scalar-value.js' export function composeScalar( schema: Schema, tagName: string | null, - token: SourceToken | BlockScalar, + token: FlowScalar | BlockScalar, onError: (offset: number, message: string, warning?: boolean) => void ) { let type: diff --git a/src/compose/flow-scalar-value.ts b/src/compose/flow-scalar-value.ts index 7eac0978..c93bc66b 100644 --- a/src/compose/flow-scalar-value.ts +++ b/src/compose/flow-scalar-value.ts @@ -1,8 +1,8 @@ import { Type } from '../constants.js' -import type { SourceToken } from '../parse/parser.js' +import type { FlowScalar } from '../parse/parser.js' export function flowScalarValue( - { type, source }: SourceToken, + { type, source }: FlowScalar, onError: (offset: number, message: string) => void, onType: (type: Type.PLAIN | Type.QUOTE_DOUBLE | Type.QUOTE_SINGLE) => void ) { diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 18a3e6a4..e61b63ac 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -2,10 +2,10 @@ import { Lexer } from './lexer.js' import { SourceTokenType, prettyToken, tokenType } from './token-type.js' export interface SourceToken { - type: SourceTokenType + type: Exclude indent: number source: string - end?: Token[] + end?: SourceToken[] } export interface ErrorToken { @@ -21,13 +21,23 @@ export interface Directive { export interface Document { type: 'document' - start: Token[] + offset: number + start: SourceToken[] value?: Token - end?: Token[] + end?: SourceToken[] +} + +export interface FlowScalar { + type: 'alias' | 'scalar' | 'single-quoted-scalar' | 'double-quoted-scalar' + offset: number + indent: number + source: string + end?: SourceToken[] } export interface BlockScalar { type: 'block-scalar' + offset: number indent: number props: Token[] source?: string @@ -35,21 +45,29 @@ export interface BlockScalar { export interface BlockMap { type: 'block-map' + offset: number indent: number items: Array< - | { start: Token[]; key?: never; sep?: never; value?: never } - | { start: Token[]; key: Token | null; sep: Token[]; value?: Token } + | { start: SourceToken[]; key?: never; sep?: never; value?: never } + | { + start: SourceToken[] + key: Token | null + sep: SourceToken[] + value?: Token + } > } export interface BlockSequence { type: 'block-seq' + offset: number indent: number items: Array<{ start: SourceToken[]; value?: Token }> } export interface FlowCollection { type: 'flow-collection' + offset: number indent: number start: Token items: Token[] @@ -61,6 +79,7 @@ export type Token = | ErrorToken | Directive | Document + | FlowScalar | BlockScalar | BlockMap | BlockSequence @@ -155,7 +174,7 @@ export class Parser { if (this.atNewLine) this.indent += source.length break case 'doc-mode': - break + return default: this.atNewLine = false } @@ -250,7 +269,11 @@ export class Parser { return case 'doc-mode': case 'doc-start': { - const doc: Document = { type: 'document', start: [] } + const doc: Document = { + type: 'document', + offset: this.offset, + start: [] + } if (this.type === 'doc-start') doc.start.push(this.sourceToken) this.stack.push(doc) return @@ -284,9 +307,9 @@ export class Parser { } } - scalar(scalar: SourceToken) { + scalar(scalar: FlowScalar) { if (this.type === 'map-value-ind') { - let sep: Token[] + let sep: SourceToken[] if (scalar.end) { sep = scalar.end sep.push(this.sourceToken) @@ -294,6 +317,7 @@ export class Parser { } else sep = [this.sourceToken] const map: BlockMap = { type: 'block-map', + offset: scalar.offset, indent: scalar.indent, items: [{ start: [], key: scalar, sep }] } @@ -356,6 +380,7 @@ export class Parser { else this.stack.push({ type: 'block-map', + offset: this.offset, indent: this.indent, items: [{ start: [this.sourceToken] }] }) @@ -368,6 +393,7 @@ export class Parser { else if (it.sep.some(tok => tok.type === 'map-value-ind')) this.stack.push({ type: 'block-map', + offset: this.offset, indent: this.indent, items: [{ start: [], key: null, sep: [this.sourceToken] }] }) @@ -377,12 +403,13 @@ export class Parser { case 'alias': case 'scalar': case 'single-quoted-scalar': - case 'double-quoted-scalar': - if (it.value) - map.items.push({ start: [], key: this.sourceToken, sep: [] }) - else if (it.sep) this.stack.push(this.sourceToken) - else Object.assign(it, { key: this.sourceToken, sep: [] }) + case 'double-quoted-scalar': { + const fs = this.flowScalar(this.type) + if (it.value) map.items.push({ start: [], key: fs, sep: [] }) + else if (it.sep) this.stack.push(fs) + else Object.assign(it, { key: fs, sep: [] }) return + } default: { const bv = this.startBlockValue() @@ -433,11 +460,14 @@ export class Parser { case 'map-value-ind': case 'anchor': case 'tag': + fc.items.push(this.sourceToken) + return + case 'alias': case 'scalar': case 'single-quoted-scalar': case 'double-quoted-scalar': - fc.items.push(this.sourceToken) + fc.items.push(this.flowScalar(this.type)) return case 'flow-map-end': @@ -452,58 +482,73 @@ export class Parser { this.step() } + flowScalar( + type: 'alias' | 'scalar' | 'single-quoted-scalar' | 'double-quoted-scalar' + ) { + if (this.onNewLine) { + let nl = this.source.indexOf('\n') + 1 + while (nl !== 0) { + this.onNewLine(this.offset + nl) + nl = this.source.indexOf('\n', nl) + 1 + } + } + return { + type, + offset: this.offset, + indent: this.indent, + source: this.source + } as FlowScalar + } + startBlockValue() { - const st = this.sourceToken switch (this.type) { case 'alias': case 'scalar': case 'single-quoted-scalar': case 'double-quoted-scalar': - if (this.onNewLine) { - let nl = this.source.indexOf('\n') + 1 - while (nl !== 0) { - this.onNewLine(this.offset + nl) - nl = this.source.indexOf('\n', nl) + 1 - } - } - return st + return this.flowScalar(this.type) case 'block-scalar-header': return { type: 'block-scalar', + offset: this.offset, indent: this.indent, - props: [st] + props: [this.sourceToken] } as BlockScalar case 'flow-map-start': case 'flow-seq-start': return { type: 'flow-collection', + offset: this.offset, indent: this.indent, - start: st, + start: this.sourceToken, items: [] } as FlowCollection case 'seq-item-ind': return { type: 'block-seq', + offset: this.offset, indent: this.indent, - items: [{ start: [st] }] + items: [{ start: [this.sourceToken] }] } as BlockSequence case 'explicit-key-ind': return { type: 'block-map', + offset: this.offset, indent: this.indent, - items: [{ start: [st] }] + items: [{ start: [this.sourceToken] }] } as BlockMap case 'map-value-ind': return { type: 'block-map', + offset: this.offset, indent: this.indent, - items: [{ start: [], key: null, sep: [st] }] + items: [{ start: [], key: null, sep: [this.sourceToken] }] } as BlockMap } return null } - lineEnd(token: SourceToken | Document) { + lineEnd(token: SourceToken | Document | FlowScalar) { switch (this.type) { case 'space': case 'comment': From b2ca63fbe37c530aec3dcde7b9a89918f1890b04 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 15:50:58 +0300 Subject: [PATCH 17/89] Refactor scalar composition to also determine node [start, end) range --- src/compose/compose-scalar.ts | 54 +++---------- ...calar-value.ts => resolve-block-scalar.ts} | 78 ++++++++++++------- ...scalar-value.ts => resolve-flow-scalar.ts} | 61 +++++++++++---- 3 files changed, 111 insertions(+), 82 deletions(-) rename src/compose/{block-scalar-value.ts => resolve-block-scalar.ts} (70%) rename src/compose/{flow-scalar-value.ts => resolve-flow-scalar.ts} (75%) diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index d53a994d..2e8512ba 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -1,9 +1,8 @@ import { Scalar } from '../ast/index.js' -import type { Type } from '../constants.js' import type { Schema } from '../doc/Schema.js' -import type { BlockScalar, FlowScalar, Token } from '../parse/parser.js' -import { blockScalarValue } from './block-scalar-value.js' -import { flowScalarValue } from './flow-scalar-value.js' +import type { BlockScalar, FlowScalar } from '../parse/parser.js' +import { resolveBlockScalar } from './resolve-block-scalar.js' +import { resolveFlowScalar } from './resolve-flow-scalar.js' export function composeScalar( schema: Schema, @@ -11,20 +10,11 @@ export function composeScalar( token: FlowScalar | BlockScalar, onError: (offset: number, message: string, warning?: boolean) => void ) { - let type: - | Type.BLOCK_FOLDED - | Type.BLOCK_LITERAL - | Type.PLAIN - | Type.QUOTE_DOUBLE - | Type.QUOTE_SINGLE - | null = null - const onType = (t: typeof type) => { - type = t - } - const value = + const { value, type, comment, length } = token.type === 'block-scalar' - ? blockScalarValue(token, onError, onType) - : flowScalarValue(token, onError, onType) + ? resolveBlockScalar(token, onError) + : resolveFlowScalar(token, onError) + const tag = findScalarTagByName(schema, value, tagName, onError) || findScalarTagByTest(schema, value, token.type === 'scalar') || @@ -38,14 +28,11 @@ export function composeScalar( onError(0, error.message) scalar = new Scalar(value) } + scalar.range = [token.offset, token.offset + length] if (type) scalar.type = type if (tagName) scalar.tag = tagName - if (tag && tag.format) scalar.format = tag.format - - // attach comments - const cs = token.type === 'block-scalar' ? token.props : token.end - const cv = cs && commentValue(cs) - if (cv) scalar.comment = cv + if (tag?.format) scalar.format = tag.format + if (comment) scalar.comment = comment return scalar } @@ -65,8 +52,7 @@ function findScalarTagByName( else return tag } } - for (const tag of matchWithTest) - if (tag.test && tag.test.test(value)) return tag + for (const tag of matchWithTest) if (tag.test?.test(value)) return tag const kt = schema.knownTags[tagName] if (kt) { // Ensure that the known tag is available for stringifying, @@ -81,24 +67,8 @@ function findScalarTagByName( function findScalarTagByTest(schema: Schema, value: string, apply: boolean) { if (apply) { for (const tag of schema.tags) { - if (tag.default && tag.test && tag.test.test(value)) return tag + if (tag.default && tag.test?.test(value)) return tag } } return null } - -function commentValue(tokens: Token[]) { - let comment = '' - let hasComment = false - let sep = '' - for (const c of tokens) { - if (c.type === 'comment') { - const cb = c.source.substring(1) - if (!hasComment) comment = cb - else comment += sep + cb - hasComment = true - sep = '' - } else if (hasComment && c.type === 'newline') sep += c.source - } - return comment -} diff --git a/src/compose/block-scalar-value.ts b/src/compose/resolve-block-scalar.ts similarity index 70% rename from src/compose/block-scalar-value.ts rename to src/compose/resolve-block-scalar.ts index cea63ff8..2dbe3d94 100644 --- a/src/compose/block-scalar-value.ts +++ b/src/compose/resolve-block-scalar.ts @@ -1,14 +1,19 @@ import { Type } from '../constants.js' import type { BlockScalar, Token } from '../parse/parser.js' -export function blockScalarValue( +export function resolveBlockScalar( scalar: BlockScalar, - onError: (offset: number, message: string) => void, - onType: (type: Type.BLOCK_LITERAL | Type.BLOCK_FOLDED) => void -) { + onError: (offset: number, message: string) => void +): { + value: string + type: Type.BLOCK_FOLDED | Type.BLOCK_LITERAL | null + comment: string + length: number +} { const header = parseBlockScalarHeader(scalar.props, onError) - if (!header || !scalar.source) return '' - const lines = splitLines(scalar.source) + if (!header) return { value: '', type: null, comment: '', length: 0 } + const type = header.mode === '>' ? Type.BLOCK_FOLDED : Type.BLOCK_LITERAL + const lines = scalar.source ? splitLines(scalar.source) : [] // determine the end of content & start of chomping let chompStart = lines.length @@ -17,10 +22,16 @@ export function blockScalarValue( else break // shortcut for empty contents - if (chompStart === 0) { - if (header.chomp === '+') - return lines.map(line => line[0]).join('\n') + '\n' - else return header.chomp === '-' ? '' : '\n' + if (!scalar.source || chompStart === 0) { + const value = + header.chomp === '+' + ? lines.map(line => line[0]).join('\n') + '\n' + : header.chomp === '-' + ? '' + : '\n' + let length = header.length + if (scalar.source) length += scalar.source.length + return { value, type, comment: header.comment, length } } // find the indentation level to trim from start @@ -45,16 +56,14 @@ export function blockScalarValue( offset += indent.length + content.length + 1 } - let res = '' + let value = '' let sep = '' let prevMoreIndented = false // leading whitespace is kept intact for (let i = 0; i < contentStart; ++i) - res += lines[i][0].slice(trimIndent) + '\n' + value += lines[i][0].slice(trimIndent) + '\n' - const folded = header.mode === '>' - onType(folded ? Type.BLOCK_FOLDED : Type.BLOCK_LITERAL) for (let i = contentStart; i < chompStart; ++i) { let [indent, content] = lines[i] offset += indent.length + content.length + 1 @@ -64,7 +73,7 @@ export function blockScalarValue( if (indent.length < trimIndent) { if (content === '') { // empty line - if (sep === '\n') res += '\n' + if (sep === '\n') value += '\n' else sep = '\n' continue } else { @@ -77,35 +86,42 @@ export function blockScalarValue( } } - if (folded) { + if (type === Type.BLOCK_FOLDED) { if (!indent || indent.length === trimIndent) { - res += sep + content + value += sep + content sep = ' ' prevMoreIndented = false } else { // more-indented content within a folded block if (sep === ' ') sep = '\n' else if (!prevMoreIndented && sep === '\n') sep = '\n\n' - res += sep + indent.slice(trimIndent) + content + value += sep + indent.slice(trimIndent) + content sep = '\n' prevMoreIndented = true } } else { // literal - res += sep + indent.slice(trimIndent) + content + value += sep + indent.slice(trimIndent) + content sep = '\n' } } switch (header.chomp) { case '-': - return res + break case '+': for (let i = chompStart; i < lines.length; ++i) - res += '\n' + lines[i][0].slice(trimIndent) - return res[res.length - 1] === '\n' ? res : res + '\n' + value += '\n' + lines[i][0].slice(trimIndent) + if (value[value.length - 1] !== '\n') value += '\n' default: - return res + '\n' + value += '\n' + } + + return { + value, + type, + comment: header.comment, + length: header.length + scalar.source.length } } @@ -133,24 +149,32 @@ function parseBlockScalarHeader( } if (error !== -1) onError(error, `Block scalar header includes extra characters: ${source}`) + let comment = '' let length = source.length for (let i = 1; i < props.length; ++i) { const token = props[i] switch (token.type) { case 'space': - case 'comment': case 'newline': length += token.source.length break + case 'comment': + length += token.source.length + comment = token.source.substring(1) + break + case 'error': + onError(length, token.message) + length += token.source.length + break default: { const message = `Unexpected token in block scalar header: ${token.type}` onError(length, message) - const ts = (token as any).source || '' - if (typeof ts === 'string') length += ts.length + const ts = (token as any).source + if (ts && typeof ts === 'string') length += ts.length } } } - return { mode, indent, chomp, length } + return { mode, indent, chomp, comment, length } } /** @returns Array of lines split up as `[indent, content]` */ diff --git a/src/compose/flow-scalar-value.ts b/src/compose/resolve-flow-scalar.ts similarity index 75% rename from src/compose/flow-scalar-value.ts rename to src/compose/resolve-flow-scalar.ts index c93bc66b..ef476d79 100644 --- a/src/compose/flow-scalar-value.ts +++ b/src/compose/resolve-flow-scalar.ts @@ -1,26 +1,61 @@ import { Type } from '../constants.js' import type { FlowScalar } from '../parse/parser.js' -export function flowScalarValue( - { type, source }: FlowScalar, - onError: (offset: number, message: string) => void, - onType: (type: Type.PLAIN | Type.QUOTE_DOUBLE | Type.QUOTE_SINGLE) => void -) { +export function resolveFlowScalar( + { type, source, end }: FlowScalar, + onError: (offset: number, message: string) => void +): { + value: string + type: Type.PLAIN | Type.QUOTE_DOUBLE | Type.QUOTE_SINGLE | null + comment: string + length: number +} { + let _type: Type.PLAIN | Type.QUOTE_DOUBLE | Type.QUOTE_SINGLE + let value: string switch (type) { case 'scalar': - onType(Type.PLAIN) - return plainValue(source, onError) + _type = Type.PLAIN + value = plainValue(source, onError) + break case 'single-quoted-scalar': - onType(Type.QUOTE_SINGLE) - return singleQuotedValue(source, onError) + _type = Type.QUOTE_SINGLE + value = singleQuotedValue(source, onError) + break case 'double-quoted-scalar': - onType(Type.QUOTE_DOUBLE) - return doubleQuotedValue(source, onError) + _type = Type.QUOTE_DOUBLE + value = doubleQuotedValue(source, onError) + break + + default: + onError(0, `Expected a flow scalar value, but found: ${type}`) + return { + value: '', + type: null, + comment: '', + length: source.length + } } - onError(0, `Expected a flow scalar value, but found: ${type}`) - return '' + + let comment = '' + let length = source.length + if (end) { + let hasComment = false + let sep = '' + for (const token of end) { + if (token.type === 'comment') { + const cb = token.source.substring(1) + if (!hasComment) comment = cb + else comment += sep + cb + hasComment = true + sep = '' + } else if (hasComment && token.type === 'newline') sep += token.source + length += token.source.length + } + } + + return { value, type: _type, comment, length } } function plainValue( From 54e9d2e2ffb65adb25a3d698aa7d1c8964c1352e Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 18:30:17 +0300 Subject: [PATCH 18/89] Refactor compose error callback to always use offset from beginning of stream --- src/compose/compose-scalar.ts | 9 ++++--- src/compose/resolve-block-scalar.ts | 16 ++++++------ src/compose/resolve-end.ts | 21 ++++++++++++++++ src/compose/resolve-flow-scalar.ts | 38 +++++++++-------------------- 4 files changed, 46 insertions(+), 38 deletions(-) create mode 100644 src/compose/resolve-end.ts diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index 2e8512ba..3f860725 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -8,8 +8,9 @@ export function composeScalar( schema: Schema, tagName: string | null, token: FlowScalar | BlockScalar, - onError: (offset: number, message: string, warning?: boolean) => void + onError: (offset: number, message: string) => void ) { + const { offset } = token const { value, type, comment, length } = token.type === 'block-scalar' ? resolveBlockScalar(token, onError) @@ -22,13 +23,13 @@ export function composeScalar( let scalar: Scalar try { - const res = tag ? tag.resolve(value, message => onError(0, message)) : value + const res = tag ? tag.resolve(value, msg => onError(offset, msg)) : value scalar = res instanceof Scalar ? res : new Scalar(res) } catch (error) { - onError(0, error.message) + onError(offset, error.message) scalar = new Scalar(value) } - scalar.range = [token.offset, token.offset + length] + scalar.range = [offset, offset + length] if (type) scalar.type = type if (tagName) scalar.tag = tagName if (tag?.format) scalar.format = tag.format diff --git a/src/compose/resolve-block-scalar.ts b/src/compose/resolve-block-scalar.ts index 2dbe3d94..8cf7d183 100644 --- a/src/compose/resolve-block-scalar.ts +++ b/src/compose/resolve-block-scalar.ts @@ -1,5 +1,5 @@ import { Type } from '../constants.js' -import type { BlockScalar, Token } from '../parse/parser.js' +import type { BlockScalar } from '../parse/parser.js' export function resolveBlockScalar( scalar: BlockScalar, @@ -10,7 +10,7 @@ export function resolveBlockScalar( comment: string length: number } { - const header = parseBlockScalarHeader(scalar.props, onError) + const header = parseBlockScalarHeader(scalar, onError) if (!header) return { value: '', type: null, comment: '', length: 0 } const type = header.mode === '>' ? Type.BLOCK_FOLDED : Type.BLOCK_LITERAL const lines = scalar.source ? splitLines(scalar.source) : [] @@ -36,7 +36,7 @@ export function resolveBlockScalar( // find the indentation level to trim from start let trimIndent = scalar.indent + header.indent - let offset = header.length + let offset = scalar.offset + header.length let contentStart = 0 for (let i = 0; i < chompStart; ++i) { const [indent, content] = lines[i] @@ -126,11 +126,11 @@ export function resolveBlockScalar( } function parseBlockScalarHeader( - props: Token[], + { offset, props }: BlockScalar, onError: (offset: number, message: string) => void ) { if (props[0].type !== 'block-scalar-header') { - onError(0, 'Block scalar header not found') + onError(offset, 'Block scalar header not found') return null } const { source } = props[0] @@ -144,7 +144,7 @@ function parseBlockScalarHeader( else { const n = Number(ch) if (!indent && n) indent = n - else if (error === -1) error = i + else if (error === -1) error = offset + i } } if (error !== -1) @@ -163,12 +163,12 @@ function parseBlockScalarHeader( comment = token.source.substring(1) break case 'error': - onError(length, token.message) + onError(offset + length, token.message) length += token.source.length break default: { const message = `Unexpected token in block scalar header: ${token.type}` - onError(length, message) + onError(offset + length, message) const ts = (token as any).source if (ts && typeof ts === 'string') length += ts.length } diff --git a/src/compose/resolve-end.ts b/src/compose/resolve-end.ts new file mode 100644 index 00000000..43b244b1 --- /dev/null +++ b/src/compose/resolve-end.ts @@ -0,0 +1,21 @@ +import { SourceToken } from '../parse/parser.js' + +export function resolveEnd(end: SourceToken[] | undefined) { + let comment = '' + let length = 0 + if (end) { + let hasComment = false + let sep = '' + for (const token of end) { + if (token.type === 'comment') { + const cb = token.source.substring(1) + if (!hasComment) comment = cb + else comment += sep + cb + hasComment = true + sep = '' + } else if (hasComment && token.type === 'newline') sep += token.source + length += token.source.length + } + } + return { comment, length } +} diff --git a/src/compose/resolve-flow-scalar.ts b/src/compose/resolve-flow-scalar.ts index ef476d79..a2b6776d 100644 --- a/src/compose/resolve-flow-scalar.ts +++ b/src/compose/resolve-flow-scalar.ts @@ -1,8 +1,9 @@ import { Type } from '../constants.js' import type { FlowScalar } from '../parse/parser.js' +import { resolveEnd } from './resolve-end.js' export function resolveFlowScalar( - { type, source, end }: FlowScalar, + { offset, type, source, end }: FlowScalar, onError: (offset: number, message: string) => void ): { value: string @@ -12,24 +13,25 @@ export function resolveFlowScalar( } { let _type: Type.PLAIN | Type.QUOTE_DOUBLE | Type.QUOTE_SINGLE let value: string + const _onError = (rel: number, msg: string) => onError(offset + rel, msg) switch (type) { case 'scalar': _type = Type.PLAIN - value = plainValue(source, onError) + value = plainValue(source, _onError) break case 'single-quoted-scalar': _type = Type.QUOTE_SINGLE - value = singleQuotedValue(source, onError) + value = singleQuotedValue(source, _onError) break case 'double-quoted-scalar': _type = Type.QUOTE_DOUBLE - value = doubleQuotedValue(source, onError) + value = doubleQuotedValue(source, _onError) break default: - onError(0, `Expected a flow scalar value, but found: ${type}`) + onError(offset, `Expected a flow scalar value, but found: ${type}`) return { value: '', type: null, @@ -38,29 +40,13 @@ export function resolveFlowScalar( } } - let comment = '' - let length = source.length - if (end) { - let hasComment = false - let sep = '' - for (const token of end) { - if (token.type === 'comment') { - const cb = token.source.substring(1) - if (!hasComment) comment = cb - else comment += sep + cb - hasComment = true - sep = '' - } else if (hasComment && token.type === 'newline') sep += token.source - length += token.source.length - } - } - - return { value, type: _type, comment, length } + const { comment, length } = resolveEnd(end) + return { value, type: _type, comment, length: source.length + length } } function plainValue( source: string, - onError: (offset: number, message: string) => void + onError: (relOffset: number, message: string) => void ) { switch (source[0]) { case '\t': @@ -78,7 +64,7 @@ function plainValue( function singleQuotedValue( source: string, - onError: (offset: number, message: string) => void + onError: (relOffset: number, message: string) => void ) { if (source[source.length - 1] !== "'") onError(source.length, "Missing closing 'quote") @@ -104,7 +90,7 @@ function foldLines(source: string) { function doubleQuotedValue( source: string, - onError: (offset: number, message: string) => void + onError: (relOffset: number, message: string) => void ) { let res = '' for (let i = 1; i < source.length - 1; ++i) { From 3e61321b8debb584ea01342109ce2a373bb9171e Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 4 Oct 2020 18:31:04 +0300 Subject: [PATCH 19/89] Start on supporting collection composition, with block sequences --- src/compose/compose-block-seq.ts | 86 ++++++++++++++++++++++++++++++++ src/compose/compose-node.ts | 48 ++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 src/compose/compose-block-seq.ts create mode 100644 src/compose/compose-node.ts diff --git a/src/compose/compose-block-seq.ts b/src/compose/compose-block-seq.ts new file mode 100644 index 00000000..42d2ee5c --- /dev/null +++ b/src/compose/compose-block-seq.ts @@ -0,0 +1,86 @@ +import { YAMLSeq } from '../ast/index.js' +import type { Document } from '../doc/Document.js' +import type { BlockSequence } from '../parse/parser.js' +import { composeNode } from './compose-node.js' + +export function composeBlockSeq( + doc: Document.Parsed, + { items, offset }: BlockSequence, + onError: (offset: number, message: string, warning?: boolean) => void +) { + const start = offset + const seq = new YAMLSeq(doc.schema) + for (const { start, value } of items) { + let spaceBefore = false + let comment = '' + let hasComment = false + let sep = '' + let anchor = '' + let tagName = '' + let hasSeparator = false + for (const token of start) { + switch (token.type) { + case 'space': + break + case 'comment': { + const cb = token.source.substring(1) + if (!hasComment) { + if (sep) spaceBefore = true + comment = cb + } else comment += sep + cb + hasComment = true + sep = '' + break + } + case 'newline': + sep += token.source + break + case 'anchor': + anchor = token.source.substring(1) + break + case 'tag': + tagName = token.source // FIXME + break + case 'seq-item-ind': + // Could here handle preceding comments differently + hasSeparator = true + break + default: + onError( + offset, + `Unexpected token before sequence item: ${token.type}` + ) + } + if (token.source) offset += token.source.length + } + if (!hasSeparator) { + if (anchor || tagName || value) { + onError(offset, 'Sequence item without - indicator') + } else { + // TODO: assert being at last item? + if (comment) seq.comment = comment + continue + } + } + // FIXME: recursive anchors + const node = composeNode(doc, value || null, tagName, onError) + if (comment) { + if (spaceBefore) node.spaceBefore = true + if (value) node.commentBefore = comment + else node.comment = comment + } else if (sep) node.spaceBefore = true + if (anchor) doc.anchors.setAnchor(node, anchor) + if (node.range) offset = node.range[1] + else { + // FIXME: remove once verified never happens + onError(offset, 'Resolved child node has no range') + if (value) { + if ('offset' in value) offset = value.offset + if ('source' in value && value.source) offset += value.source.length + } + } + seq.items.push(node) + } + seq.range = [start, offset] + return seq +} diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts new file mode 100644 index 00000000..b55e4c18 --- /dev/null +++ b/src/compose/compose-node.ts @@ -0,0 +1,48 @@ +import { Alias, Node } from '../ast/index.js' +import type { Document } from '../doc/Document.js' +import type { FlowScalar, Token } from '../parse/parser.js' +import { composeBlockSeq } from './compose-block-seq.js' +import { composeScalar } from './compose-scalar.js' +import { resolveEnd } from './resolve-end.js' + +export function composeNode( + doc: Document.Parsed, + token: Token | null, + tagName: string | null, + onError: (offset: number, message: string, warning?: boolean) => void +) { + if (!token) token = { type: 'scalar', offset: 0, indent: 0, source: '' } + switch (token.type) { + case 'alias': + return composeAlias(doc.anchors, token, onError) + case 'scalar': + case 'single-quoted-scalar': + case 'double-quoted-scalar': + case 'block-scalar': + return composeScalar(doc.schema, tagName, token, onError) + case 'block-seq': + // FIXME: anchor on collection needs to resolve in the collection + return composeBlockSeq(doc, token, onError) + } + return new Node() +} + +function composeAlias( + anchors: Document.Anchors, + { offset, source, end }: FlowScalar, + onError: (offset: number, message: string, warning?: boolean) => void +) { + const name = source.substring(1) + const src = anchors.getNode(name) + + // FIXME: Lazy resolution for circular references + const alias = new Alias(src as Node) + + if (src) anchors._cstAliases.push(alias) + else onError(offset, `Aliased anchor not found: ${name}`) + + const { comment, length } = resolveEnd(end) + alias.range = [offset, offset + source.length + length] + if (comment) alias.comment = comment + return alias +} From 37004ceb0208d4179c8646250b1c87590037b536 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 5 Oct 2020 00:36:36 +0300 Subject: [PATCH 20/89] Add initial block mapping composition, refactoring around it --- src/compose/compose-block-map.ts | 83 ++++++++++++++++++++++++++++++++ src/compose/compose-block-seq.ts | 65 +++++-------------------- src/compose/compose-node.ts | 44 +++++++++++------ src/compose/resolve-props.ts | 59 +++++++++++++++++++++++ src/parse/parser.ts | 3 +- 5 files changed, 185 insertions(+), 69 deletions(-) create mode 100644 src/compose/compose-block-map.ts create mode 100644 src/compose/resolve-props.ts diff --git a/src/compose/compose-block-map.ts b/src/compose/compose-block-map.ts new file mode 100644 index 00000000..b2b8f8fc --- /dev/null +++ b/src/compose/compose-block-map.ts @@ -0,0 +1,83 @@ +import { Pair, YAMLMap } from '../ast/index.js' +import type { Document } from '../doc/Document.js' +import type { BlockMap } from '../parse/parser.js' +import { composeNode } from './compose-node.js' +import { resolveProps } from './resolve-props.js' + +export function composeBlockMap( + doc: Document.Parsed, + { items, offset }: BlockMap, + anchor: string | null, + onError: (offset: number, message: string, warning?: boolean) => void +) { + const start = offset + const map = new YAMLMap(doc.schema) + if (anchor) doc.anchors.setAnchor(map, anchor) + const onRelError = (rel: number, msg: string) => onError(offset + rel, msg) + + for (const { start, key, sep, value } of items) { + // key properties + const keyProps = resolveProps(start, 'explicit-key-ind', onRelError) + if (!keyProps.found) { + // implicit key + if (keyProps.anchor || keyProps.tagName || sep) { + // FIXME: check single-line + // FIXME: check 1024 chars + } else { + // TODO: assert being at last item? + if (keyProps.comment) { + if (map.comment) map.comment += '\n' + keyProps.comment + else map.comment = keyProps.comment + } + continue + } + } + offset += keyProps.length + + // key value + const keyStart = offset + const kt = key || { type: 'scalar', offset, indent: -1, source: '' } + const keyNode = composeNode(doc, kt, keyProps, onError) + if (keyNode.range) offset = keyNode.range[1] + else { + // FIXME: remove once verified never happens + onError(offset, 'Resolved child node has no range') + if (key) { + if ('offset' in key) offset = key.offset + if ('source' in key && key.source) offset += key.source.length + } + } + + // value properties + const valueProps = resolveProps(sep || [], 'map-value-ind', onRelError) + offset += valueProps.length + + if (valueProps.found) { + // value value + const vt = value || { type: 'scalar', offset, indent: -1, source: '' } + const valueNode = composeNode(doc, vt, valueProps, onError) + if (valueNode.range) offset = valueNode.range[1] + else { + // FIXME: remove once verified never happens + onError(offset, 'Resolved child node has no range') + if (value) { + if ('offset' in value) offset = value.offset + if ('source' in value && value.source) offset += value.source.length + } + } + + map.items.push(new Pair(keyNode, valueNode)) + } else { + // key with no value + if (!keyProps.found) + onError(keyStart, 'Implicit map keys need to be followed by map values') + if (valueProps.comment) { + if (map.comment) map.comment += '\n' + valueProps.comment + else map.comment = valueProps.comment + } + map.items.push(new Pair(keyNode)) + } + } + map.range = [start, offset] + return map +} diff --git a/src/compose/compose-block-seq.ts b/src/compose/compose-block-seq.ts index 42d2ee5c..7ff66f1a 100644 --- a/src/compose/compose-block-seq.ts +++ b/src/compose/compose-block-seq.ts @@ -2,74 +2,33 @@ import { YAMLSeq } from '../ast/index.js' import type { Document } from '../doc/Document.js' import type { BlockSequence } from '../parse/parser.js' import { composeNode } from './compose-node.js' +import { resolveProps } from './resolve-props.js' export function composeBlockSeq( doc: Document.Parsed, { items, offset }: BlockSequence, + anchor: string | null, onError: (offset: number, message: string, warning?: boolean) => void ) { const start = offset const seq = new YAMLSeq(doc.schema) + if (anchor) doc.anchors.setAnchor(seq, anchor) for (const { start, value } of items) { - let spaceBefore = false - let comment = '' - let hasComment = false - let sep = '' - let anchor = '' - let tagName = '' - let hasSeparator = false - for (const token of start) { - switch (token.type) { - case 'space': - break - case 'comment': { - const cb = token.source.substring(1) - if (!hasComment) { - if (sep) spaceBefore = true - comment = cb - } else comment += sep + cb - hasComment = true - sep = '' - break - } - case 'newline': - sep += token.source - break - case 'anchor': - anchor = token.source.substring(1) - break - case 'tag': - tagName = token.source // FIXME - break - case 'seq-item-ind': - // Could here handle preceding comments differently - hasSeparator = true - break - default: - onError( - offset, - `Unexpected token before sequence item: ${token.type}` - ) - } - if (token.source) offset += token.source.length - } - if (!hasSeparator) { - if (anchor || tagName || value) { + const props = resolveProps(start, 'seq-item-ind', (o, m) => + onError(offset + o, m) + ) + offset += props.length + if (!props.found) { + if (props.anchor || props.tagName || value) { onError(offset, 'Sequence item without - indicator') } else { // TODO: assert being at last item? - if (comment) seq.comment = comment + if (props.comment) seq.comment = props.comment continue } } - // FIXME: recursive anchors - const node = composeNode(doc, value || null, tagName, onError) - if (comment) { - if (spaceBefore) node.spaceBefore = true - if (value) node.commentBefore = comment - else node.comment = comment - } else if (sep) node.spaceBefore = true - if (anchor) doc.anchors.setAnchor(node, anchor) + const token = value || { type: 'scalar', offset, indent: -1, source: '' } + const node = composeNode(doc, token, props, onError) if (node.range) offset = node.range[1] else { // FIXME: remove once verified never happens diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts index b55e4c18..0c6c07f8 100644 --- a/src/compose/compose-node.ts +++ b/src/compose/compose-node.ts @@ -1,30 +1,51 @@ import { Alias, Node } from '../ast/index.js' import type { Document } from '../doc/Document.js' import type { FlowScalar, Token } from '../parse/parser.js' +import { composeBlockMap } from './compose-block-map.js' import { composeBlockSeq } from './compose-block-seq.js' import { composeScalar } from './compose-scalar.js' import { resolveEnd } from './resolve-end.js' +import type { Props } from './resolve-props.js' export function composeNode( doc: Document.Parsed, - token: Token | null, - tagName: string | null, + token: Token, + { spaceBefore, comment, anchor, tagName }: Props, onError: (offset: number, message: string, warning?: boolean) => void ) { - if (!token) token = { type: 'scalar', offset: 0, indent: 0, source: '' } + let node: Node switch (token.type) { case 'alias': - return composeAlias(doc.anchors, token, onError) + node = composeAlias(doc.anchors, token, onError) + if (anchor || tagName) + onError(token.offset, 'An alias node must not specify any properties') + break case 'scalar': case 'single-quoted-scalar': case 'double-quoted-scalar': case 'block-scalar': - return composeScalar(doc.schema, tagName, token, onError) + node = composeScalar(doc.schema, tagName, token, onError) + if (anchor) doc.anchors.setAnchor(node, anchor) + break + case 'block-map': + node = composeBlockMap(doc, token, anchor, onError) + break case 'block-seq': - // FIXME: anchor on collection needs to resolve in the collection - return composeBlockSeq(doc, token, onError) + node = composeBlockSeq(doc, token, anchor, onError) + break + default: + onError( + 'offset' in token ? token.offset : -1, + `Unsupporten token type: ${token.type}` + ) + return new Node() } - return new Node() + if (spaceBefore) node.spaceBefore = true + if (comment) { + if (token.type === 'scalar' && token.source === '') node.comment = comment + else node.commentBefore = comment + } + return node } function composeAlias( @@ -34,13 +55,8 @@ function composeAlias( ) { const name = source.substring(1) const src = anchors.getNode(name) - - // FIXME: Lazy resolution for circular references + if (!src) onError(offset, `Aliased anchor not found: ${name}`) const alias = new Alias(src as Node) - - if (src) anchors._cstAliases.push(alias) - else onError(offset, `Aliased anchor not found: ${name}`) - const { comment, length } = resolveEnd(end) alias.range = [offset, offset + source.length + length] if (comment) alias.comment = comment diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts new file mode 100644 index 00000000..ad1dc8b6 --- /dev/null +++ b/src/compose/resolve-props.ts @@ -0,0 +1,59 @@ +import { SourceToken } from '../parse/parser' + +export interface Props { + found: boolean + spaceBefore: boolean + comment: string + anchor: string + tagName: string + length: number +} + +export function resolveProps( + start: SourceToken[], + indicator: 'explicit-key-ind' | 'map-value-ind' | 'seq-item-ind', + onError: (relOffset: number, message: string) => void +) { + let length = 0 + let spaceBefore = false + let comment = '' + let hasComment = false + let sep = '' + let anchor = '' + let tagName = '' + let found = false + for (const token of start) { + switch (token.type) { + case 'space': + break + case 'comment': { + const cb = token.source.substring(1) + if (!hasComment) { + if (sep) spaceBefore = true + comment = cb + } else comment += sep + cb + hasComment = true + sep = '' + break + } + case 'newline': + sep += token.source + break + case 'anchor': + anchor = token.source.substring(1) + break + case 'tag': + tagName = token.source // FIXME + break + case indicator: + // Could here handle preceding comments differently + found = true + break + default: + onError(length, `Unexpected ${token.type} token`) + } + if (token.source) length += token.source.length + } + if (!comment && sep) spaceBefore = true + return { found, spaceBefore, comment, anchor, tagName, length } +} diff --git a/src/parse/parser.ts b/src/parse/parser.ts index e61b63ac..6adf2b5d 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -5,7 +5,6 @@ export interface SourceToken { type: Exclude indent: number source: string - end?: SourceToken[] } export interface ErrorToken { @@ -548,7 +547,7 @@ export class Parser { return null } - lineEnd(token: SourceToken | Document | FlowScalar) { + lineEnd(token: Document | FlowScalar) { switch (this.type) { case 'space': case 'comment': From ba7ef7be1aa549aa2e56eeeb7d2dfc550eba1660 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 5 Oct 2020 10:20:15 +0300 Subject: [PATCH 21/89] lexer: Fix for lines without value --- src/parse/lexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 868d7e1b..c27279ef 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -274,7 +274,7 @@ export class Lexer { switch (line[n]) { case undefined: case '#': - this.pushCount(line.length) + this.pushCount(line.length - n) this.pushNewline() return this.parseLineStart() case '{': From 94942b5e0372d35f532facf376b8f5fe0f90f78f Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 5 Oct 2020 10:31:39 +0300 Subject: [PATCH 22/89] parser: Start new document on doc-start after content --- src/parse/parser.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 6adf2b5d..faf828c6 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -285,7 +285,17 @@ export class Parser { document(doc: Document) { if (doc.value) return this.lineEnd(doc) switch (this.type) { - case 'doc-start': + case 'doc-start': { + const hasContent = doc.start.some( + ({ type }) => + type === 'doc-start' || type === 'anchor' || type === 'tag' + ) + if (hasContent) { + this.pop() + this.step() + } else doc.start.push(this.sourceToken) + return + } case 'anchor': case 'tag': case 'space': From 153f816958a128490aee6a3abd34bf429627a1fc Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 5 Oct 2020 11:37:46 +0300 Subject: [PATCH 23/89] Add document composition, with test wrapper at compose/parse-docs --- src/compose/compose-block-map.ts | 5 ++-- src/compose/compose-block-seq.ts | 4 +--- src/compose/compose-doc.ts | 39 ++++++++++++++++++++++++++++++++ src/compose/parse-docs.ts | 33 +++++++++++++++++++++++++++ src/compose/resolve-props.ts | 11 ++++++--- src/compose/stream-directives.ts | 2 +- 6 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 src/compose/compose-doc.ts create mode 100644 src/compose/parse-docs.ts diff --git a/src/compose/compose-block-map.ts b/src/compose/compose-block-map.ts index b2b8f8fc..4a42ed1f 100644 --- a/src/compose/compose-block-map.ts +++ b/src/compose/compose-block-map.ts @@ -13,11 +13,10 @@ export function composeBlockMap( const start = offset const map = new YAMLMap(doc.schema) if (anchor) doc.anchors.setAnchor(map, anchor) - const onRelError = (rel: number, msg: string) => onError(offset + rel, msg) for (const { start, key, sep, value } of items) { // key properties - const keyProps = resolveProps(start, 'explicit-key-ind', onRelError) + const keyProps = resolveProps(start, 'explicit-key-ind', offset, onError) if (!keyProps.found) { // implicit key if (keyProps.anchor || keyProps.tagName || sep) { @@ -49,7 +48,7 @@ export function composeBlockMap( } // value properties - const valueProps = resolveProps(sep || [], 'map-value-ind', onRelError) + const valueProps = resolveProps(sep || [], 'map-value-ind', offset, onError) offset += valueProps.length if (valueProps.found) { diff --git a/src/compose/compose-block-seq.ts b/src/compose/compose-block-seq.ts index 7ff66f1a..b12fed17 100644 --- a/src/compose/compose-block-seq.ts +++ b/src/compose/compose-block-seq.ts @@ -14,9 +14,7 @@ export function composeBlockSeq( const seq = new YAMLSeq(doc.schema) if (anchor) doc.anchors.setAnchor(seq, anchor) for (const { start, value } of items) { - const props = resolveProps(start, 'seq-item-ind', (o, m) => - onError(offset + o, m) - ) + const props = resolveProps(start, 'seq-item-ind', offset, onError) offset += props.length if (!props.found) { if (props.anchor || props.tagName || value) { diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts new file mode 100644 index 00000000..74ceb9b2 --- /dev/null +++ b/src/compose/compose-doc.ts @@ -0,0 +1,39 @@ +import { Document } from '../doc/Document.js' +import type { Options } from '../options.js' +import type * as Parser from '../parse/parser.js' +import { composeNode } from './compose-node.js' +import { resolveEnd } from './resolve-end.js' +import { resolveProps } from './resolve-props.js' +import { StreamDirectives } from './stream-directives.js' + +export function composeDoc( + options: Options | null, + directives: StreamDirectives, + { offset, start, value, end }: Parser.Document, + onError: (offset: number, message: string, warning?: boolean) => void +) { + const doc = new Document(undefined, options || undefined) as Document.Parsed + doc.version = directives.yaml.version + doc.setSchema() // FIXME: always do this in the constructor + + const props = resolveProps(start, 'doc-start', offset, onError) + if (props.found) doc.directivesEndMarker = true + + let to = offset + props.length + const token = value || { type: 'scalar', offset: to, indent: -1, source: '' } + doc.contents = composeNode(doc, token, props, onError) + if (doc.contents.range) to = doc.contents.range[1] + else { + // FIXME: remove once verified never happens + onError(to, 'Resolved child node has no range') + if (value) { + if ('offset' in value) to = value.offset + if ('source' in value && value.source) to += value.source.length + } + } + const { comment, length } = resolveEnd(end) + if (comment) doc.comment = comment + + doc.range = [offset, to + length] + return doc +} diff --git a/src/compose/parse-docs.ts b/src/compose/parse-docs.ts new file mode 100644 index 00000000..3179b42a --- /dev/null +++ b/src/compose/parse-docs.ts @@ -0,0 +1,33 @@ +import { Document } from '../doc/Document' +import { Parser } from '../parse/parser' +import { composeDoc } from './compose-doc' +import { StreamDirectives } from './stream-directives' + +export function parseDocs(source: string) { + const directives = new StreamDirectives() + const docs: Document.Parsed[] = [] + const lines: number[] = [] + + const onError = (offset: number, message: string, warning?: boolean) => { + console.error(warning ? '???' : '!!!', { offset, message }) + } + + const parser = new Parser( + token => { + switch (token.type) { + case 'directive': + directives.add(token.source, onError) + break + case 'document': + docs.push(composeDoc(null, directives, token, onError)) + break + default: + console.log('###', token) + } + }, + n => lines.push(n) + ) + parser.parse(source) + + return docs +} diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index ad1dc8b6..2e3575e5 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -11,8 +11,13 @@ export interface Props { export function resolveProps( start: SourceToken[], - indicator: 'explicit-key-ind' | 'map-value-ind' | 'seq-item-ind', - onError: (relOffset: number, message: string) => void + indicator: + | 'doc-start' + | 'explicit-key-ind' + | 'map-value-ind' + | 'seq-item-ind', + offset: number, + onError: (offset: number, message: string) => void ) { let length = 0 let spaceBefore = false @@ -50,7 +55,7 @@ export function resolveProps( found = true break default: - onError(length, `Unexpected ${token.type} token`) + onError(offset + length, `Unexpected ${token.type} token`) } if (token.source) length += token.source.length } diff --git a/src/compose/stream-directives.ts b/src/compose/stream-directives.ts index 1f089c35..35ad8335 100644 --- a/src/compose/stream-directives.ts +++ b/src/compose/stream-directives.ts @@ -1,6 +1,6 @@ export class StreamDirectives { tags: Record = { '!!': 'tag:yaml.org,2002:' } - yaml: { version: '1.1' | '1.2' } = { version: '1.2' } + yaml: { version: '1.1' | '1.2' | undefined } = { version: undefined } static from(src: StreamDirectives) { const res = new StreamDirectives() From 99f27ce34fe2a4e04548659ab97f9fe4cc86ae65 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 24 Oct 2020 11:14:08 +0300 Subject: [PATCH 24/89] Overload token arg of composeNode() for empty node support --- src/compose/compose-block-map.ts | 6 ++---- src/compose/compose-block-seq.ts | 3 +-- src/compose/compose-doc.ts | 3 +-- src/compose/compose-node.ts | 21 +++++++++++++++++++-- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/compose/compose-block-map.ts b/src/compose/compose-block-map.ts index 4a42ed1f..4ab6d61e 100644 --- a/src/compose/compose-block-map.ts +++ b/src/compose/compose-block-map.ts @@ -35,8 +35,7 @@ export function composeBlockMap( // key value const keyStart = offset - const kt = key || { type: 'scalar', offset, indent: -1, source: '' } - const keyNode = composeNode(doc, kt, keyProps, onError) + const keyNode = composeNode(doc, key || offset, keyProps, onError) if (keyNode.range) offset = keyNode.range[1] else { // FIXME: remove once verified never happens @@ -53,8 +52,7 @@ export function composeBlockMap( if (valueProps.found) { // value value - const vt = value || { type: 'scalar', offset, indent: -1, source: '' } - const valueNode = composeNode(doc, vt, valueProps, onError) + const valueNode = composeNode(doc, value || offset, valueProps, onError) if (valueNode.range) offset = valueNode.range[1] else { // FIXME: remove once verified never happens diff --git a/src/compose/compose-block-seq.ts b/src/compose/compose-block-seq.ts index b12fed17..f02a480b 100644 --- a/src/compose/compose-block-seq.ts +++ b/src/compose/compose-block-seq.ts @@ -25,8 +25,7 @@ export function composeBlockSeq( continue } } - const token = value || { type: 'scalar', offset, indent: -1, source: '' } - const node = composeNode(doc, token, props, onError) + const node = composeNode(doc, value || offset, props, onError) if (node.range) offset = node.range[1] else { // FIXME: remove once verified never happens diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index 74ceb9b2..32a539fd 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -20,8 +20,7 @@ export function composeDoc( if (props.found) doc.directivesEndMarker = true let to = offset + props.length - const token = value || { type: 'scalar', offset: to, indent: -1, source: '' } - doc.contents = composeNode(doc, token, props, onError) + doc.contents = composeNode(doc, value || to, props, onError) if (doc.contents.range) to = doc.contents.range[1] else { // FIXME: remove once verified never happens diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts index 0c6c07f8..64195a70 100644 --- a/src/compose/compose-node.ts +++ b/src/compose/compose-node.ts @@ -9,10 +9,13 @@ import type { Props } from './resolve-props.js' export function composeNode( doc: Document.Parsed, - token: Token, - { spaceBefore, comment, anchor, tagName }: Props, + token: Token | number, + props: Props, onError: (offset: number, message: string, warning?: boolean) => void ) { + if (typeof token === 'number') + return composeEmptyNode(doc, token, props, onError) + const { spaceBefore, comment, anchor, tagName } = props let node: Node switch (token.type) { case 'alias': @@ -48,6 +51,20 @@ export function composeNode( return node } +function composeEmptyNode( + doc: Document.Parsed, + offset: number, + { spaceBefore, comment, anchor, tagName }: Props, + onError: (offset: number, message: string, warning?: boolean) => void +) { + const token: FlowScalar = { type: 'scalar', offset, indent: -1, source: '' } + const node = composeScalar(doc.schema, tagName, token, onError) + if (anchor) doc.anchors.setAnchor(node, anchor) + if (spaceBefore) node.spaceBefore = true + if (comment) node.comment = comment + return node +} + function composeAlias( anchors: Document.Anchors, { offset, source, end }: FlowScalar, From 4a1d99bb5c483e45438e0b8d6f8c0c5a13a2c187 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 5 Dec 2020 12:23:20 +0200 Subject: [PATCH 25/89] Prop resolution fixes --- src/compose/compose-node.ts | 8 +++++++- src/compose/resolve-props.ts | 12 +++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts index 64195a70..34e0417b 100644 --- a/src/compose/compose-node.ts +++ b/src/compose/compose-node.ts @@ -5,7 +5,13 @@ import { composeBlockMap } from './compose-block-map.js' import { composeBlockSeq } from './compose-block-seq.js' import { composeScalar } from './compose-scalar.js' import { resolveEnd } from './resolve-end.js' -import type { Props } from './resolve-props.js' + +export interface Props { + spaceBefore: boolean + comment: string + anchor: string + tagName: string +} export function composeNode( doc: Document.Parsed, diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index 2e3575e5..f2052eee 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -1,14 +1,5 @@ import { SourceToken } from '../parse/parser' -export interface Props { - found: boolean - spaceBefore: boolean - comment: string - anchor: string - tagName: string - length: number -} - export function resolveProps( start: SourceToken[], indicator: @@ -45,9 +36,12 @@ export function resolveProps( sep += token.source break case 'anchor': + if (anchor) + onError(offset + length, 'A node can have at most one anchor') anchor = token.source.substring(1) break case 'tag': + if (tagName) onError(offset + length, 'A node can have at most one tag') tagName = token.source // FIXME break case indicator: From 217f9fe24889be00641d31143466645d96bc9db2 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 5 Dec 2020 12:23:47 +0200 Subject: [PATCH 26/89] Add initial flow collection composition --- src/compose/compose-flow-collection.ts | 151 +++++++++++++++++++++++++ src/compose/compose-node.ts | 4 + src/parse/parser.ts | 6 +- 3 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 src/compose/compose-flow-collection.ts diff --git a/src/compose/compose-flow-collection.ts b/src/compose/compose-flow-collection.ts new file mode 100644 index 00000000..c928394d --- /dev/null +++ b/src/compose/compose-flow-collection.ts @@ -0,0 +1,151 @@ +import { Node, Pair, YAMLMap, YAMLSeq } from '../ast/index.js' +import type { Document } from '../doc/Document.js' +import type { FlowCollection, SourceToken } from '../parse/parser.js' +import { composeNode } from './compose-node.js' + +export function composeFlowCollection( + doc: Document.Parsed, + fc: FlowCollection, + _anchor: string | null, + onError: (offset: number, message: string, warning?: boolean) => void +) { + let offset = fc.offset + const isMap = fc.start.source === '{' + const coll = isMap ? new YAMLMap(doc.schema) : new YAMLSeq(doc.schema) + if (_anchor) doc.anchors.setAnchor(coll, _anchor) + + let key: Node | null = null + let value: Node | null = null + + let spaceBefore = false + let comment = '' + let hasComment = false + let newlines = '' + let anchor = '' + let tagName = '' + + // let atExplicitKey = false + let atValueEnd = false + + function resetProps() { + spaceBefore = false + comment = '' + hasComment = false + newlines = '' + anchor = '' + tagName = '' + // atExplicitKey = false + atValueEnd = false + } + + function addItem() { + if (value) { + if (hasComment) value.comment = comment + } else { + const props = { spaceBefore, comment, anchor, tagName } + value = composeNode(doc, offset, props, onError) + } + if (isMap) { + const pair = key ? new Pair(key, value) : new Pair(value) + coll.items.push(pair) + } else { + const seq = coll as YAMLSeq + if (key) { + const map = new YAMLMap(doc.schema) + map.items.push(new Pair(key, value)) + seq.items.push(map) + } else seq.items.push(value) + } + resetProps() + } + + for (const token of fc.items) { + let isSourceToken = true + switch (token.type) { + case 'space': + break + case 'comment': + const cb = token.source.substring(1) + if (!hasComment) { + if (newlines) spaceBefore = true + comment = cb + } else comment += newlines + cb + hasComment = true + newlines = '' + break + case 'newline': + if (atValueEnd) { + if (hasComment) { + let node = coll.items[coll.items.length - 1] + if (node instanceof Pair) node = node.value || node.key + if (node instanceof Node) node.comment = comment + else onError(offset, 'Error adding trailing comment to node') + comment = '' + hasComment = false + } + atValueEnd = false + } else newlines += token.source + break + case 'anchor': + if (anchor) onError(offset, 'A node can have at most one anchor') + anchor = token.source.substring(1) + break + case 'tag': + if (tagName) onError(offset, 'A node can have at most one tag') + tagName = token.source // FIXME + break + case 'explicit-key-ind': + // atExplicitKey = true + if (anchor || tagName) + onError(offset, 'Anchors and tags must be after the ? indicator') + break + case 'map-value-ind': { + if (key) { + if (value) { + onError(offset, 'Missing {} around pair used as mapping key') + const map = new YAMLMap(doc.schema) + map.items.push(new Pair(key, value)) + key = map + value = null + } // else explicit key + } else if (value) { + key = value + value = null + } else { + const props = { spaceBefore, comment, anchor, tagName } + key = composeNode(doc, offset, props, onError) // empty node + resetProps() + } + if (hasComment) { + key.comment = comment + comment = '' + hasComment = false + } + break + } + case 'comma': + addItem() + atValueEnd = true + key = null + value = null + break + default: { + if (value) onError(offset, 'Missing , between flow collection items') + const props = { spaceBefore, comment, anchor, tagName } + value = composeNode(doc, token, props, onError) + if (value.range) offset = value.range[1] + else { + // FIXME: remove once verified never happens + onError(offset, 'Resolved child node has no range') + if ('offset' in token) offset = token.offset + if ('source' in token && token.source) offset += token.source.length + } + isSourceToken = false + } + } + if (isSourceToken) offset += (token as SourceToken).source.length + } + if (key || value) addItem() + coll.range = [fc.offset, offset] + return coll +} diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts index 34e0417b..be90e403 100644 --- a/src/compose/compose-node.ts +++ b/src/compose/compose-node.ts @@ -3,6 +3,7 @@ import type { Document } from '../doc/Document.js' import type { FlowScalar, Token } from '../parse/parser.js' import { composeBlockMap } from './compose-block-map.js' import { composeBlockSeq } from './compose-block-seq.js' +import { composeFlowCollection } from './compose-flow-collection.js' import { composeScalar } from './compose-scalar.js' import { resolveEnd } from './resolve-end.js' @@ -42,6 +43,9 @@ export function composeNode( case 'block-seq': node = composeBlockSeq(doc, token, anchor, onError) break + case 'flow-collection': + node = composeFlowCollection(doc, token, anchor, onError) + break default: onError( 'offset' in token ? token.offset : -1, diff --git a/src/parse/parser.ts b/src/parse/parser.ts index faf828c6..c8112018 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -68,9 +68,9 @@ export interface FlowCollection { type: 'flow-collection' offset: number indent: number - start: Token - items: Token[] - end?: Token + start: SourceToken + items: Array + end?: SourceToken } export type Token = From f3a51f8ac56d370bee6b4bde17bb00895720e88b Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Wed, 9 Dec 2020 07:43:25 +0200 Subject: [PATCH 27/89] Fix config to allow TS sources in tests --- babel.config.js | 5 ++++- jest.config.js | 3 ++- package-lock.json | 37 +++++++++++++++++++++++++++++++++++++ package.json | 2 ++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/babel.config.js b/babel.config.js index 12d88914..d707f22d 100644 --- a/babel.config.js +++ b/babel.config.js @@ -6,4 +6,7 @@ module.exports = { } if (process.env.NODE_ENV === 'test') - module.exports.presets = [['@babel/env', { targets: { node: 'current' } }]] + module.exports.presets = [ + ['@babel/env', { targets: { node: 'current' } }], + '@babel/preset-typescript' + ] diff --git a/jest.config.js b/jest.config.js index fde23bdc..ce346215 100644 --- a/jest.config.js +++ b/jest.config.js @@ -32,8 +32,9 @@ switch (process.env.npm_lifecycle_event) { module.exports = { collectCoverageFrom: ['src/**/*.js'], moduleNameMapper, + resolver: 'jest-ts-webcompat-resolver', testEnvironment: 'node', testMatch: ['**/tests/**/*.js'], testPathIgnorePatterns, - transform: { '/(src|tests)/.*\\.js$': 'babel-jest' } + transform: { '/(src|tests)/.*\\.(js|ts)$': 'babel-jest' } } diff --git a/package-lock.json b/package-lock.json index c370f8de..cf82b55c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1137,6 +1137,15 @@ "@babel/helper-plugin-utils": "^7.10.4" } }, + "@babel/plugin-syntax-typescript": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.12.1.tgz", + "integrity": "sha512-UZNEcCY+4Dp9yYRCAHrHDU+9ZXLYaY9MgBXSRLkB9WjYFRR6quJBumfVrEkUxrePPBwFcpWfNKXqVRQQtm7mMA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, "@babel/plugin-transform-arrow-functions": { "version": "7.12.1", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.12.1.tgz", @@ -1431,6 +1440,17 @@ "@babel/helper-plugin-utils": "^7.10.4" } }, + "@babel/plugin-transform-typescript": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.12.1.tgz", + "integrity": "sha512-VrsBByqAIntM+EYMqSm59SiMEf7qkmI9dqMt6RbD/wlwueWmYcI0FFK5Fj47pP6DRZm+3teXjosKlwcZJ5lIMw==", + "dev": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.12.1", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-typescript": "^7.12.1" + } + }, "@babel/plugin-transform-unicode-escapes": { "version": "7.12.1", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.12.1.tgz", @@ -1556,6 +1576,17 @@ "esutils": "^2.0.2" } }, + "@babel/preset-typescript": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.12.7.tgz", + "integrity": "sha512-nOoIqIqBmHBSEgBXWR4Dv/XBehtIFcw9PqZw6rFYuKrzsZmOQm3PR5siLBnKZFEsDb03IegG8nSjU/iXXXYRmw==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-validator-option": "^7.12.1", + "@babel/plugin-transform-typescript": "^7.12.1" + } + }, "@babel/runtime": { "version": "7.12.5", "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.12.5.tgz", @@ -5572,6 +5603,12 @@ } } }, + "jest-ts-webcompat-resolver": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/jest-ts-webcompat-resolver/-/jest-ts-webcompat-resolver-1.0.0.tgz", + "integrity": "sha512-BFoaU7LeYqZNnTYEr6iMRf87xdCQntNc/Wk8YpzDBcuz+CIZ0JsTtzuMAMnKiEgTRTC1wRWLUo2RlVjVijBcHQ==", + "dev": true + }, "jest-util": { "version": "26.6.2", "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-26.6.2.tgz", diff --git a/package.json b/package.json index 98950256..d1b58ae8 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "@babel/core": "^7.12.10", "@babel/plugin-proposal-class-properties": "^7.12.1", "@babel/preset-env": "^7.12.11", + "@babel/preset-typescript": "^7.12.7", "@rollup/plugin-babel": "^5.2.2", "babel-eslint": "^10.1.0", "babel-jest": "^26.6.3", @@ -86,6 +87,7 @@ "eslint-config-prettier": "^7.1.0", "fast-check": "^2.10.0", "jest": "^26.6.3", + "jest-ts-webcompat-resolver": "^1.0.0", "prettier": "^2.2.1", "rollup": "^2.35.1", "typescript": "^4.1.3" From 1b037f02410f90a8f94aea311fdf566b318588db Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Wed, 9 Dec 2020 07:46:58 +0200 Subject: [PATCH 28/89] Add minimal scaffold for using stream parser via old API --- src/compose/compose-doc.ts | 4 ++-- src/compose/parse-docs.ts | 43 ++++++++++++++++++++++++++++++++++---- src/errors.d.ts | 8 +++++++ src/errors.js | 12 ++++++++--- src/index.js | 21 ++++++------------- src/parse/parser.ts | 2 +- 6 files changed, 65 insertions(+), 25 deletions(-) diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index 32a539fd..b04cb131 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -7,12 +7,12 @@ import { resolveProps } from './resolve-props.js' import { StreamDirectives } from './stream-directives.js' export function composeDoc( - options: Options | null, + options: Options | undefined, directives: StreamDirectives, { offset, start, value, end }: Parser.Document, onError: (offset: number, message: string, warning?: boolean) => void ) { - const doc = new Document(undefined, options || undefined) as Document.Parsed + const doc = new Document(undefined, options) as Document.Parsed doc.version = directives.yaml.version doc.setSchema() // FIXME: always do this in the constructor diff --git a/src/compose/parse-docs.ts b/src/compose/parse-docs.ts index 3179b42a..b4708fa8 100644 --- a/src/compose/parse-docs.ts +++ b/src/compose/parse-docs.ts @@ -1,15 +1,32 @@ import { Document } from '../doc/Document' +import { YAMLParseError, YAMLWarning } from '../errors' +import type { Options } from '../options' import { Parser } from '../parse/parser' import { composeDoc } from './compose-doc' import { StreamDirectives } from './stream-directives' -export function parseDocs(source: string) { +export function parseDocs(source: string, options?: Options) { const directives = new StreamDirectives() const docs: Document.Parsed[] = [] const lines: number[] = [] + let comment = '' + let errors: YAMLParseError[] = [] + let warnings: YAMLWarning[] = [] const onError = (offset: number, message: string, warning?: boolean) => { - console.error(warning ? '???' : '!!!', { offset, message }) + warning + ? warnings.push(new YAMLWarning(offset, message)) + : errors.push(new YAMLParseError(offset, message)) + } + const decorate = (doc: Document.Parsed) => { + if (comment) doc.commentBefore = comment.trimRight() + comment = '' + + doc.errors = errors + errors = [] + + doc.warnings = warnings + warnings = [] } const parser = new Parser( @@ -18,8 +35,26 @@ export function parseDocs(source: string) { case 'directive': directives.add(token.source, onError) break - case 'document': - docs.push(composeDoc(null, directives, token, onError)) + case 'document': { + const doc = composeDoc(options, directives, token, onError) + decorate(doc) + docs.push(doc) + break + } + case 'comment': + comment += token.source.substring(1) + break + case 'newline': + if (comment) comment += token.source + break + case 'error': { + const msg = token.source + ? `${token.message}: ${JSON.stringify(token.source)}` + : token.message + errors.push(new YAMLParseError(-1, msg)) + break + } + case 'space': break default: console.log('###', token) diff --git a/src/errors.d.ts b/src/errors.d.ts index 2c184e68..87a7dbfa 100644 --- a/src/errors.d.ts +++ b/src/errors.d.ts @@ -8,11 +8,13 @@ interface LinePos { export class YAMLError extends Error { name: + | 'YAMLParseError' | 'YAMLReferenceError' | 'YAMLSemanticError' | 'YAMLSyntaxError' | 'YAMLWarning' message: string + offset?: number source?: CST.Node nodeType?: Type @@ -27,6 +29,11 @@ export class YAMLError extends Error { makePretty(): void } +export class YAMLParseError extends YAMLError { + name: 'YAMLParseError' + constructor(source: Node | number, message: string) +} + export class YAMLReferenceError extends YAMLError { name: 'YAMLReferenceError' } @@ -41,4 +48,5 @@ export class YAMLSyntaxError extends YAMLError { export class YAMLWarning extends YAMLError { name: 'YAMLWarning' + constructor(source: Node | number, message: string) } diff --git a/src/errors.js b/src/errors.js index 872123a4..0f4f3610 100644 --- a/src/errors.js +++ b/src/errors.js @@ -4,12 +4,12 @@ import { Range } from './cst/Range.js' export class YAMLError extends Error { constructor(name, source, message) { - if (!message || !(source instanceof Node)) - throw new Error(`Invalid arguments for new ${name}`) + if (!message) throw new Error(`Invalid arguments for new ${name}`) super() this.name = name this.message = message - this.source = source + if (source instanceof Node) this.source = source + else if (typeof source === 'number') this.offset = source } makePretty() { @@ -38,6 +38,12 @@ export class YAMLError extends Error { } } +export class YAMLParseError extends YAMLError { + constructor(source, message) { + super('YAMLParseError', source, message) + } +} + export class YAMLReferenceError extends YAMLError { constructor(source, message) { super('YAMLReferenceError', source, message) diff --git a/src/index.js b/src/index.js index c76c33bf..e95197a7 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,4 @@ +import { parseDocs } from './compose/parse-docs.js' import { LogLevel } from './constants.js' import { parse as parseCST } from './cst/parse.js' import { Document } from './doc/Document.js' @@ -7,28 +8,18 @@ import { warn } from './log.js' export { defaultOptions, scalarOptions } from './options.js' export { Document, parseCST } -export function parseAllDocuments(src, options) { - const stream = [] - let prev - for (const cstDoc of parseCST(src)) { - const doc = new Document(undefined, null, options) - doc.parse(cstDoc, prev) - stream.push(doc) - prev = doc - } - return stream -} +export const parseAllDocuments = parseDocs export function parseDocument(src, options) { - const cst = parseCST(src) - const doc = new Document(cst[0], null, options) + const docs = parseDocs(src, options) + const doc = docs[0] if ( - cst.length > 1 && + docs.length > 1 && LogLevel.indexOf(doc.options.logLevel) >= LogLevel.ERROR ) { const errMsg = 'Source contains multiple documents; please use YAML.parseAllDocuments()' - doc.errors.unshift(new YAMLSemanticError(cst[1], errMsg)) + doc.errors.unshift(new YAMLSemanticError(-1, errMsg)) } return doc } diff --git a/src/parse/parser.ts b/src/parse/parser.ts index c8112018..2c2ad7b3 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -141,7 +141,7 @@ export class Parser { /** Advance the parser by the `source` of one lexical token. */ token(source: string) { this.source = source - console.log('>', prettyToken(source)) + //console.log('>', prettyToken(source)) if (this.atScalar) { this.atScalar = false From c21b37532cd1329ba07d5225e161593f446e8cac Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 14 Dec 2020 11:19:08 +0200 Subject: [PATCH 29/89] Add directives to Document, use for proper tag names --- src/compose/compose-block-map.ts | 16 ++++++++++++++-- src/compose/compose-block-seq.ts | 8 +++++++- src/compose/compose-doc.ts | 9 ++++++++- src/compose/compose-flow-collection.ts | 6 ++++-- src/compose/resolve-props.ts | 8 ++++++-- src/compose/stream-directives.ts | 10 +++++----- src/doc/Document.d.ts | 5 +++++ 7 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/compose/compose-block-map.ts b/src/compose/compose-block-map.ts index 4ab6d61e..8eabbb03 100644 --- a/src/compose/compose-block-map.ts +++ b/src/compose/compose-block-map.ts @@ -16,7 +16,13 @@ export function composeBlockMap( for (const { start, key, sep, value } of items) { // key properties - const keyProps = resolveProps(start, 'explicit-key-ind', offset, onError) + const keyProps = resolveProps( + doc.directives, + start, + 'explicit-key-ind', + offset, + onError + ) if (!keyProps.found) { // implicit key if (keyProps.anchor || keyProps.tagName || sep) { @@ -47,7 +53,13 @@ export function composeBlockMap( } // value properties - const valueProps = resolveProps(sep || [], 'map-value-ind', offset, onError) + const valueProps = resolveProps( + doc.directives, + sep || [], + 'map-value-ind', + offset, + onError + ) offset += valueProps.length if (valueProps.found) { diff --git a/src/compose/compose-block-seq.ts b/src/compose/compose-block-seq.ts index f02a480b..fb8c2d29 100644 --- a/src/compose/compose-block-seq.ts +++ b/src/compose/compose-block-seq.ts @@ -14,7 +14,13 @@ export function composeBlockSeq( const seq = new YAMLSeq(doc.schema) if (anchor) doc.anchors.setAnchor(seq, anchor) for (const { start, value } of items) { - const props = resolveProps(start, 'seq-item-ind', offset, onError) + const props = resolveProps( + doc.directives, + start, + 'seq-item-ind', + offset, + onError + ) offset += props.length if (!props.found) { if (props.anchor || props.tagName || value) { diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index b04cb131..a38ce21f 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -13,10 +13,17 @@ export function composeDoc( onError: (offset: number, message: string, warning?: boolean) => void ) { const doc = new Document(undefined, options) as Document.Parsed + doc.directives = StreamDirectives.from(directives) doc.version = directives.yaml.version doc.setSchema() // FIXME: always do this in the constructor - const props = resolveProps(start, 'doc-start', offset, onError) + const props = resolveProps( + doc.directives, + start, + 'doc-start', + offset, + onError + ) if (props.found) doc.directivesEndMarker = true let to = offset + props.length diff --git a/src/compose/compose-flow-collection.ts b/src/compose/compose-flow-collection.ts index c928394d..f666f0ab 100644 --- a/src/compose/compose-flow-collection.ts +++ b/src/compose/compose-flow-collection.ts @@ -90,10 +90,12 @@ export function composeFlowCollection( if (anchor) onError(offset, 'A node can have at most one anchor') anchor = token.source.substring(1) break - case 'tag': + case 'tag': { if (tagName) onError(offset, 'A node can have at most one tag') - tagName = token.source // FIXME + const tn = doc.directives.tagName(token.source, m => onError(offset, m)) + if (tn) tagName = tn break + } case 'explicit-key-ind': // atExplicitKey = true if (anchor || tagName) diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index f2052eee..0574016e 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -1,6 +1,8 @@ import { SourceToken } from '../parse/parser' +import { StreamDirectives } from './stream-directives' export function resolveProps( + directives: StreamDirectives, start: SourceToken[], indicator: | 'doc-start' @@ -40,10 +42,12 @@ export function resolveProps( onError(offset + length, 'A node can have at most one anchor') anchor = token.source.substring(1) break - case 'tag': + case 'tag': { if (tagName) onError(offset + length, 'A node can have at most one tag') - tagName = token.source // FIXME + const tn = directives.tagName(token.source, msg => onError(offset, msg)) + if (tn) tagName = tn break + } case indicator: // Could here handle preceding comments differently found = true diff --git a/src/compose/stream-directives.ts b/src/compose/stream-directives.ts index 35ad8335..6ca298e0 100644 --- a/src/compose/stream-directives.ts +++ b/src/compose/stream-directives.ts @@ -52,22 +52,22 @@ export class StreamDirectives { * @returns Resolved tag, which may also be the non-specific tag `'!'` or a * `'!local'` tag, or `null` if unresolvable. */ - tagName(source: string, onError: (offset: number, message: string) => void) { + tagName(source: string, onError: (message: string) => void) { if (source === '!') return '!' // non-specific tag if (source[0] !== '!') { - onError(0, `Not a valid tag: ${source}`) + onError(`Not a valid tag: ${source}`) return null } if (source[1] === '<') { const verbatim = source.slice(2, -1) if (verbatim === '!' || verbatim === '!!') { - onError(0, `Verbatim tags aren't resolved, so ${source} is invalid.`) + onError(`Verbatim tags aren't resolved, so ${source} is invalid.`) return null } if (source[source.length - 1] !== '>') - onError(source.length - 1, 'Verbatim tags must end with a >') + onError('Verbatim tags must end with a >') return verbatim } @@ -76,7 +76,7 @@ export class StreamDirectives { if (prefix) return prefix + suffix if (handle === '!') return source // local tag - onError(0, `Could not resolve tag: ${source}`) + onError(`Could not resolve tag: ${source}`) return null } diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts index d3ca208a..5ad5919a 100644 --- a/src/doc/Document.d.ts +++ b/src/doc/Document.d.ts @@ -1,4 +1,5 @@ import { Alias, Collection, Merge, Node, Pair } from '../ast' +import { StreamDirectives } from '../compose/stream-directives' import { Type } from '../constants' import { CST } from '../cst' import { YAMLError, YAMLWarning } from '../errors' @@ -36,6 +37,9 @@ export class Document extends Collection { */ constructor(value?: any, options?: Options) constructor(value: any, replacer: null | Replacer, options?: Options) + + directives?: StreamDirectives + tag: never directivesEndMarker?: boolean type: Type.DOCUMENT @@ -129,6 +133,7 @@ export class Document extends Collection { export namespace Document { interface Parsed extends Document { contents: Node | null + directives: StreamDirectives /** The schema used with the document. */ schema: Schema } From ac3636192e97db12e72a77bacf8cf3c797758db4 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 14 Dec 2020 12:02:58 +0200 Subject: [PATCH 30/89] Refactor index.js as index.ts & adjust scripts accordingly --- .eslintignore | 1 + package.json | 5 ++++- rollup.dev-config.js | 7 ++++++- src/constants.d.ts | 11 ++++++++++- src/doc/Document.d.ts | 1 + src/errors.d.ts | 3 +++ src/{index.js => index.ts} | 22 +++++++++++++++------- src/log.d.ts | 4 ++++ src/options.d.ts | 7 +++++++ 9 files changed, 51 insertions(+), 10 deletions(-) rename src/{index.js => index.ts} (73%) create mode 100644 src/log.d.ts diff --git a/.eslintignore b/.eslintignore index 2341e6f6..818fb542 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,6 +1,7 @@ /browser/ /dist/ /docs-slate/ +/lib/ /package-lock.json /playground/dist/ /tests/yaml-test-suite/ diff --git a/package.json b/package.json index d1b58ae8..8c726ac3 100644 --- a/package.json +++ b/package.json @@ -50,11 +50,14 @@ "scripts": { "build": "npm run build:node && npm run build:browser", "build:browser": "rollup -c rollup.browser-config.js", + "prebuild:dev": "rollup -c rollup.dev-config.js", + "build:dev": "tsc", "build:node": "rollup -c rollup.node-config.js", "clean": "git clean -fdxe node_modules", "lint": "eslint src/", "prettier": "prettier --write .", - "start": "cross-env TRACE_LEVEL=log npm run build:node && node -i -e 'YAML=require(\".\")'", + "prestart": "tsc", + "start": "node -i -e 'YAML=require(\"./lib/index.js\")'", "test": "jest", "test:browsers": "cd playground && npm test", "test:dist": "npm run build:node && jest", diff --git a/rollup.dev-config.js b/rollup.dev-config.js index dd2fd13b..fc0ac30c 100644 --- a/rollup.dev-config.js +++ b/rollup.dev-config.js @@ -1,7 +1,12 @@ import babel from '@rollup/plugin-babel' export default { - input: ['src/index.js', 'src/test-events.js', 'src/types.js', 'src/util.js', 'src/ast/index.js'], + input: [ + 'src/ast/index.js', + 'src/doc/Document.js', + 'src/errors.js', + 'src/options.js' + ], output: { dir: 'lib', format: 'cjs', esModule: false, preserveModules: true }, plugins: [babel()] } diff --git a/src/constants.d.ts b/src/constants.d.ts index d5110c9c..380ba1e7 100644 --- a/src/constants.d.ts +++ b/src/constants.d.ts @@ -1,3 +1,13 @@ +export type LogLevelId = 'silent' | 'error' | 'warn' | 'debug' + +export interface LogLevel extends Array { + SILENT: 0 + ERROR: 1 + WARN: 2 + DEBUG: 3 +} +export const LogLevel: LogLevel + export enum Type { ALIAS = 'ALIAS', BLANK_LINE = 'BLANK_LINE', @@ -17,4 +27,3 @@ export enum Type { SEQ = 'SEQ', SEQ_ITEM = 'SEQ_ITEM' } - diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts index 5ad5919a..e4955a87 100644 --- a/src/doc/Document.d.ts +++ b/src/doc/Document.d.ts @@ -56,6 +56,7 @@ export class Document extends Collection { * The schema used with the document. Use `setSchema()` to change or * initialise. */ + options: Required schema?: Schema /** * Array of prefixes; each will have a string `handle` that diff --git a/src/errors.d.ts b/src/errors.d.ts index 87a7dbfa..657e2394 100644 --- a/src/errors.d.ts +++ b/src/errors.d.ts @@ -36,14 +36,17 @@ export class YAMLParseError extends YAMLError { export class YAMLReferenceError extends YAMLError { name: 'YAMLReferenceError' + constructor(source: Node | number, message: string) } export class YAMLSemanticError extends YAMLError { name: 'YAMLSemanticError' + constructor(source: Node | number, message: string) } export class YAMLSyntaxError extends YAMLError { name: 'YAMLSyntaxError' + constructor(source: Node | number, message: string) } export class YAMLWarning extends YAMLError { diff --git a/src/index.js b/src/index.ts similarity index 73% rename from src/index.js rename to src/index.ts index e95197a7..3664c5d9 100644 --- a/src/index.js +++ b/src/index.ts @@ -1,16 +1,16 @@ import { parseDocs } from './compose/parse-docs.js' import { LogLevel } from './constants.js' -import { parse as parseCST } from './cst/parse.js' import { Document } from './doc/Document.js' import { YAMLSemanticError } from './errors.js' import { warn } from './log.js' +import { Options } from './options.js' export { defaultOptions, scalarOptions } from './options.js' -export { Document, parseCST } +export { Document } export const parseAllDocuments = parseDocs -export function parseDocument(src, options) { +export function parseDocument(src: string, options?: Options) { const docs = parseDocs(src, options) const doc = docs[0] if ( @@ -24,7 +24,11 @@ export function parseDocument(src, options) { return doc } -export function parse(src, reviver, options) { +export function parse( + src: string, + reviver?: (key: string, value: any) => any, + options?: Options +) { if (options === undefined && reviver && typeof reviver === 'object') { options = reviver reviver = undefined @@ -40,15 +44,19 @@ export function parse(src, reviver, options) { return doc.toJS({ reviver }) } -export function stringify(value, replacer, options) { +export function stringify( + value: any, + replacer?: (key: string, value: any) => any, + options?: string | number | Options +) { if (typeof options === 'string') options = options.length if (typeof options === 'number') { const indent = Math.round(options) options = indent < 1 ? undefined : indent > 8 ? { indent: 8 } : { indent } } if (value === undefined) { - const { keepUndefined } = options || replacer || {} + const { keepUndefined } = options || (replacer as Options) || {} if (!keepUndefined) return undefined } - return new Document(value, replacer, options).toString() + return new Document(value, replacer || null, options).toString() } diff --git a/src/log.d.ts b/src/log.d.ts new file mode 100644 index 00000000..8b69740d --- /dev/null +++ b/src/log.d.ts @@ -0,0 +1,4 @@ +import type { LogLevelId } from './constants.js' + +export function debug(logLevel: LogLevelId, ...messages: any[]): void +export function warn(logLevel: LogLevelId, warning: string | Error): void diff --git a/src/options.d.ts b/src/options.d.ts index fe18e011..f76b4a2a 100644 --- a/src/options.d.ts +++ b/src/options.d.ts @@ -1,4 +1,5 @@ import { Scalar } from './ast' +import { LogLevelId } from './constants' import { Schema } from './doc/Schema' /** @@ -48,6 +49,12 @@ export interface Options extends Schema.Options { * Default: `false` */ keepUndefined?: boolean + /** + * Control the logging level during parsing + * + * Default: `'warn'` + */ + logLevel?: LogLevelId /** * When outputting JS, use Map rather than Object to represent mappings. * From 7cff090a73bc429438a5b5f1c42d1a2714fddbaa Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 14 Dec 2020 12:07:42 +0200 Subject: [PATCH 31/89] Use LOG_TOKENS and LOG_STREAM env vars to control debug prints --- src/compose/parse-docs.ts | 1 + src/parse/parser.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compose/parse-docs.ts b/src/compose/parse-docs.ts index b4708fa8..0d2ddab1 100644 --- a/src/compose/parse-docs.ts +++ b/src/compose/parse-docs.ts @@ -31,6 +31,7 @@ export function parseDocs(source: string, options?: Options) { const parser = new Parser( token => { + if (process.env.LOG_STREAM) console.dir(token, { depth: null }) switch (token.type) { case 'directive': directives.add(token.source, onError) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 2c2ad7b3..3f841ec2 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -141,7 +141,7 @@ export class Parser { /** Advance the parser by the `source` of one lexical token. */ token(source: string) { this.source = source - //console.log('>', prettyToken(source)) + if (process.env.LOG_TOKENS) console.log('|', prettyToken(source)) if (this.atScalar) { this.atScalar = false From 3443d08c83774535faf7ee9ff7350545df1c188c Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 14 Dec 2020 12:50:03 +0200 Subject: [PATCH 32/89] Add & use a new Node.Parsed interface when range is known --- src/ast/index.d.ts | 27 ++++++++++++++++++++++++++ src/compose/compose-block-map.ts | 23 +++------------------- src/compose/compose-block-seq.ts | 12 ++---------- src/compose/compose-doc.ts | 19 +++++++----------- src/compose/compose-flow-collection.ts | 17 ++++++---------- src/compose/compose-node.ts | 19 ++++++++---------- src/compose/compose-scalar.ts | 2 +- src/doc/Document.d.ts | 2 +- 8 files changed, 55 insertions(+), 66 deletions(-) diff --git a/src/ast/index.d.ts b/src/ast/index.d.ts index f203744c..350e8a5b 100644 --- a/src/ast/index.d.ts +++ b/src/ast/index.d.ts @@ -24,6 +24,12 @@ export class Node { type?: Type | Pair.Type } +export namespace Node { + interface Parsed extends Node { + range: [number, number] + } +} + export class Scalar extends Node { constructor(value: any) type?: Scalar.Type @@ -38,6 +44,9 @@ export class Scalar extends Node { toString(): string } export namespace Scalar { + interface Parsed extends Scalar { + range: [number, number] + } type Type = | Type.BLOCK_FOLDED | Type.BLOCK_LITERAL @@ -54,6 +63,12 @@ export class Alias extends Node { toString(ctx: Schema.StringifyContext): string } +export namespace Alias { + interface Parsed extends Alias { + range: [number, number] + } +} + export class Pair extends Node { constructor(key: any, value?: any) type: Pair.Type.PAIR | Pair.Type.MERGE_PAIR @@ -137,6 +152,12 @@ export class YAMLMap extends Collection { ): string } +export namespace YAMLMap { + interface Parsed extends YAMLMap { + range: [number, number] + } +} + export class YAMLSeq extends Collection { type?: Type.FLOW_SEQ | Type.SEQ delete(key: number | string | Scalar): boolean @@ -152,6 +173,12 @@ export class YAMLSeq extends Collection { ): string } +export namespace YAMLSeq { + interface Parsed extends YAMLSeq { + range: [number, number] + } +} + export namespace AST { interface NodeToJsonContext { anchors?: any[] diff --git a/src/compose/compose-block-map.ts b/src/compose/compose-block-map.ts index 8eabbb03..b057d9c7 100644 --- a/src/compose/compose-block-map.ts +++ b/src/compose/compose-block-map.ts @@ -42,15 +42,7 @@ export function composeBlockMap( // key value const keyStart = offset const keyNode = composeNode(doc, key || offset, keyProps, onError) - if (keyNode.range) offset = keyNode.range[1] - else { - // FIXME: remove once verified never happens - onError(offset, 'Resolved child node has no range') - if (key) { - if ('offset' in key) offset = key.offset - if ('source' in key && key.source) offset += key.source.length - } - } + offset = keyNode.range[1] // value properties const valueProps = resolveProps( @@ -65,16 +57,7 @@ export function composeBlockMap( if (valueProps.found) { // value value const valueNode = composeNode(doc, value || offset, valueProps, onError) - if (valueNode.range) offset = valueNode.range[1] - else { - // FIXME: remove once verified never happens - onError(offset, 'Resolved child node has no range') - if (value) { - if ('offset' in value) offset = value.offset - if ('source' in value && value.source) offset += value.source.length - } - } - + offset = valueNode.range[1] map.items.push(new Pair(keyNode, valueNode)) } else { // key with no value @@ -88,5 +71,5 @@ export function composeBlockMap( } } map.range = [start, offset] - return map + return map as YAMLMap.Parsed } diff --git a/src/compose/compose-block-seq.ts b/src/compose/compose-block-seq.ts index fb8c2d29..eea88feb 100644 --- a/src/compose/compose-block-seq.ts +++ b/src/compose/compose-block-seq.ts @@ -32,17 +32,9 @@ export function composeBlockSeq( } } const node = composeNode(doc, value || offset, props, onError) - if (node.range) offset = node.range[1] - else { - // FIXME: remove once verified never happens - onError(offset, 'Resolved child node has no range') - if (value) { - if ('offset' in value) offset = value.offset - if ('source' in value && value.source) offset += value.source.length - } - } + offset = node.range[1] seq.items.push(node) } seq.range = [start, offset] - return seq + return seq as YAMLSeq.Parsed } diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index a38ce21f..847774ee 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -26,20 +26,15 @@ export function composeDoc( ) if (props.found) doc.directivesEndMarker = true - let to = offset + props.length - doc.contents = composeNode(doc, value || to, props, onError) - if (doc.contents.range) to = doc.contents.range[1] - else { - // FIXME: remove once verified never happens - onError(to, 'Resolved child node has no range') - if (value) { - if ('offset' in value) to = value.offset - if ('source' in value && value.source) to += value.source.length - } - } + doc.contents = composeNode( + doc, + value || offset + props.length, + props, + onError + ) const { comment, length } = resolveEnd(end) if (comment) doc.comment = comment - doc.range = [offset, to + length] + doc.range = [offset, doc.contents.range[1] + length] return doc } diff --git a/src/compose/compose-flow-collection.ts b/src/compose/compose-flow-collection.ts index f666f0ab..14ab50c8 100644 --- a/src/compose/compose-flow-collection.ts +++ b/src/compose/compose-flow-collection.ts @@ -14,8 +14,8 @@ export function composeFlowCollection( const coll = isMap ? new YAMLMap(doc.schema) : new YAMLSeq(doc.schema) if (_anchor) doc.anchors.setAnchor(coll, _anchor) - let key: Node | null = null - let value: Node | null = null + let key: Node.Parsed | null = null + let value: Node.Parsed | null = null let spaceBefore = false let comment = '' @@ -107,7 +107,8 @@ export function composeFlowCollection( onError(offset, 'Missing {} around pair used as mapping key') const map = new YAMLMap(doc.schema) map.items.push(new Pair(key, value)) - key = map + map.range = [key.range[0], value.range[1]] + key = map as YAMLMap.Parsed value = null } // else explicit key } else if (value) { @@ -135,13 +136,7 @@ export function composeFlowCollection( if (value) onError(offset, 'Missing , between flow collection items') const props = { spaceBefore, comment, anchor, tagName } value = composeNode(doc, token, props, onError) - if (value.range) offset = value.range[1] - else { - // FIXME: remove once verified never happens - onError(offset, 'Resolved child node has no range') - if ('offset' in token) offset = token.offset - if ('source' in token && token.source) offset += token.source.length - } + offset = value.range[1] isSourceToken = false } } @@ -149,5 +144,5 @@ export function composeFlowCollection( } if (key || value) addItem() coll.range = [fc.offset, offset] - return coll + return coll as YAMLMap.Parsed | YAMLSeq.Parsed } diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts index be90e403..bc1c990f 100644 --- a/src/compose/compose-node.ts +++ b/src/compose/compose-node.ts @@ -8,10 +8,10 @@ import { composeScalar } from './compose-scalar.js' import { resolveEnd } from './resolve-end.js' export interface Props { - spaceBefore: boolean - comment: string - anchor: string - tagName: string + spaceBefore: boolean + comment: string + anchor: string + tagName: string } export function composeNode( @@ -23,7 +23,7 @@ export function composeNode( if (typeof token === 'number') return composeEmptyNode(doc, token, props, onError) const { spaceBefore, comment, anchor, tagName } = props - let node: Node + let node: Node.Parsed switch (token.type) { case 'alias': node = composeAlias(doc.anchors, token, onError) @@ -47,11 +47,8 @@ export function composeNode( node = composeFlowCollection(doc, token, anchor, onError) break default: - onError( - 'offset' in token ? token.offset : -1, - `Unsupporten token type: ${token.type}` - ) - return new Node() + console.log(token) + throw new Error(`Unsupporten token type: ${(token as any).type}`) } if (spaceBefore) node.spaceBefore = true if (comment) { @@ -87,5 +84,5 @@ function composeAlias( const { comment, length } = resolveEnd(end) alias.range = [offset, offset + source.length + length] if (comment) alias.comment = comment - return alias + return alias as Alias.Parsed } diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index 3f860725..9d3ea7c1 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -35,7 +35,7 @@ export function composeScalar( if (tag?.format) scalar.format = tag.format if (comment) scalar.comment = comment - return scalar + return scalar as Scalar.Parsed } function findScalarTagByName( diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts index e4955a87..7bf391eb 100644 --- a/src/doc/Document.d.ts +++ b/src/doc/Document.d.ts @@ -133,7 +133,7 @@ export class Document extends Collection { export namespace Document { interface Parsed extends Document { - contents: Node | null + contents: Node.Parsed | null directives: StreamDirectives /** The schema used with the document. */ schema: Schema From 138087c3e3d6b99d9fa43d8f0a029fe4d4ff35cf Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 14 Dec 2020 12:58:09 +0200 Subject: [PATCH 33/89] Make Collection schema not enumerable to hide it from logging --- src/ast/Collection.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ast/Collection.js b/src/ast/Collection.js index 3c60b5c1..92100185 100644 --- a/src/ast/Collection.js +++ b/src/ast/Collection.js @@ -45,7 +45,12 @@ export class Collection extends Node { constructor(schema) { super() - this.schema = schema + Object.defineProperty(this, 'schema', { + value: schema, + configurable: true, + enumerable: false, + writable: true + }) } addIn(path, value) { From 2fc9e65c4f6bd36fa4d4fb804fe77149c9acaefa Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 14 Dec 2020 19:41:40 +0200 Subject: [PATCH 34/89] Handle composition of all supported collections, not just map & seq --- src/ast/YAMLMap.js | 4 ++ src/ast/YAMLSeq.js | 4 ++ src/ast/index.d.ts | 2 + src/compose/compose-collection.ts | 69 +++++++++++++++++++ src/compose/compose-node.ts | 16 ++--- src/compose/compose-scalar.ts | 13 ++-- ...pose-block-map.ts => resolve-block-map.ts} | 2 +- ...pose-block-seq.ts => resolve-block-seq.ts} | 2 +- ...llection.ts => resolve-flow-collection.ts} | 5 +- src/constants.d.ts | 7 ++ src/tags/failsafe/map.js | 5 +- src/tags/failsafe/seq.js | 5 +- 12 files changed, 109 insertions(+), 25 deletions(-) create mode 100644 src/compose/compose-collection.ts rename src/compose/{compose-block-map.ts => resolve-block-map.ts} (98%) rename src/compose/{compose-block-seq.ts => resolve-block-seq.ts} (97%) rename src/compose/{compose-flow-collection.ts => resolve-flow-collection.ts} (97%) diff --git a/src/ast/YAMLMap.js b/src/ast/YAMLMap.js index e9eb32c7..0d877f02 100644 --- a/src/ast/YAMLMap.js +++ b/src/ast/YAMLMap.js @@ -14,6 +14,10 @@ export function findPair(items, key) { } export class YAMLMap extends Collection { + static get tagName() { + return 'tag:yaml.org,2002:map' + } + add(pair, overwrite) { if (!pair) pair = new Pair(pair) else if (!(pair instanceof Pair)) diff --git a/src/ast/YAMLSeq.js b/src/ast/YAMLSeq.js index 92fd2d51..410d333e 100644 --- a/src/ast/YAMLSeq.js +++ b/src/ast/YAMLSeq.js @@ -9,6 +9,10 @@ function asItemIndex(key) { } export class YAMLSeq extends Collection { + static get tagName() { + return 'tag:yaml.org,2002:map' + } + add(value) { this.items.push(value) } diff --git a/src/ast/index.d.ts b/src/ast/index.d.ts index 350e8a5b..8ff9d8e0 100644 --- a/src/ast/index.d.ts +++ b/src/ast/index.d.ts @@ -141,6 +141,7 @@ export class Collection extends Node { } export class YAMLMap extends Collection { + static readonly tagName: 'tag:yaml.org,2002:map' type?: Type.FLOW_MAP | Type.MAP items: Array hasAllNullValues(): boolean @@ -159,6 +160,7 @@ export namespace YAMLMap { } export class YAMLSeq extends Collection { + static readonly tagName: 'tag:yaml.org,2002:seq' type?: Type.FLOW_SEQ | Type.SEQ delete(key: number | string | Scalar): boolean get(key: number | string | Scalar, keepScalar?: boolean): any diff --git a/src/compose/compose-collection.ts b/src/compose/compose-collection.ts new file mode 100644 index 00000000..d928a688 --- /dev/null +++ b/src/compose/compose-collection.ts @@ -0,0 +1,69 @@ +import { Node, Scalar, YAMLMap, YAMLSeq } from '../ast/index.js' +import type { Document } from '../doc/Document.js' +import type { + BlockMap, + BlockSequence, + FlowCollection +} from '../parse/parser.js' +import { resolveBlockMap } from './resolve-block-map.js' +import { resolveBlockSeq } from './resolve-block-seq.js' +import { resolveFlowCollection } from './resolve-flow-collection.js' + +export function composeCollection( + doc: Document.Parsed, + token: BlockMap | BlockSequence | FlowCollection, + anchor: string | null, + tagName: string | null, + onError: (offset: number, message: string, warning?: boolean) => void +) { + let coll: YAMLMap.Parsed | YAMLSeq.Parsed + switch (token.type) { + case 'block-map': { + coll = resolveBlockMap(doc, token, anchor, onError) + break + } + case 'block-seq': { + coll = resolveBlockSeq(doc, token, anchor, onError) + break + } + case 'flow-collection': { + coll = resolveFlowCollection(doc, token, anchor, onError) + break + } + } + + if (!tagName) return coll + + // Cast needed due to: https://github.com/Microsoft/TypeScript/issues/3841 + const Coll = coll.constructor as typeof YAMLMap | typeof YAMLSeq + if (tagName === '!' || tagName === Coll.tagName) { + coll.tag = Coll.tagName + return coll + } + + let tag = doc.schema.tags.find(t => t.tag === tagName) + if (!tag) { + const kt = doc.schema.knownTags[tagName] + if (kt) { + doc.schema.tags.push(Object.assign({}, kt, { default: false })) + tag = kt + } else { + onError(coll.range[0], `Unresolved tag: ${tagName}`, true) + coll.tag = tagName + return coll + } + } + + const res = tag.resolve(coll, msg => onError(coll.range[0], msg)) + const node = + res instanceof Node + ? (res as Node.Parsed) + : (new Scalar(res) as Scalar.Parsed) + node.range = coll.range + node.tag = tagName + if (tag?.format) (node as Scalar).format = tag.format + if (anchor && node !== coll) { + // FIXME: handle anchor for non-failsafe collections + } + return node +} diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts index bc1c990f..b7edfa10 100644 --- a/src/compose/compose-node.ts +++ b/src/compose/compose-node.ts @@ -1,9 +1,7 @@ import { Alias, Node } from '../ast/index.js' import type { Document } from '../doc/Document.js' import type { FlowScalar, Token } from '../parse/parser.js' -import { composeBlockMap } from './compose-block-map.js' -import { composeBlockSeq } from './compose-block-seq.js' -import { composeFlowCollection } from './compose-flow-collection.js' +import { composeCollection } from './compose-collection.js' import { composeScalar } from './compose-scalar.js' import { resolveEnd } from './resolve-end.js' @@ -34,17 +32,12 @@ export function composeNode( case 'single-quoted-scalar': case 'double-quoted-scalar': case 'block-scalar': - node = composeScalar(doc.schema, tagName, token, onError) - if (anchor) doc.anchors.setAnchor(node, anchor) + node = composeScalar(doc, token, anchor, tagName, onError) break case 'block-map': - node = composeBlockMap(doc, token, anchor, onError) - break case 'block-seq': - node = composeBlockSeq(doc, token, anchor, onError) - break case 'flow-collection': - node = composeFlowCollection(doc, token, anchor, onError) + node = composeCollection(doc, token, anchor, tagName, onError) break default: console.log(token) @@ -65,8 +58,7 @@ function composeEmptyNode( onError: (offset: number, message: string, warning?: boolean) => void ) { const token: FlowScalar = { type: 'scalar', offset, indent: -1, source: '' } - const node = composeScalar(doc.schema, tagName, token, onError) - if (anchor) doc.anchors.setAnchor(node, anchor) + const node = composeScalar(doc, token, anchor, tagName, onError) if (spaceBefore) node.spaceBefore = true if (comment) node.comment = comment return node diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index 9d3ea7c1..d1fae1b8 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -1,13 +1,15 @@ import { Scalar } from '../ast/index.js' +import { Document } from '../doc/Document.js' import type { Schema } from '../doc/Schema.js' import type { BlockScalar, FlowScalar } from '../parse/parser.js' import { resolveBlockScalar } from './resolve-block-scalar.js' import { resolveFlowScalar } from './resolve-flow-scalar.js' export function composeScalar( - schema: Schema, - tagName: string | null, + doc: Document.Parsed, token: FlowScalar | BlockScalar, + anchor: string | null, + tagName: string | null, onError: (offset: number, message: string) => void ) { const { offset } = token @@ -17,9 +19,9 @@ export function composeScalar( : resolveFlowScalar(token, onError) const tag = - findScalarTagByName(schema, value, tagName, onError) || - findScalarTagByTest(schema, value, token.type === 'scalar') || - findScalarTagByName(schema, value, '!', onError) + findScalarTagByName(doc.schema, value, tagName, onError) || + findScalarTagByTest(doc.schema, value, token.type === 'scalar') || + findScalarTagByName(doc.schema, value, '!', onError) let scalar: Scalar try { @@ -35,6 +37,7 @@ export function composeScalar( if (tag?.format) scalar.format = tag.format if (comment) scalar.comment = comment + if (anchor) doc.anchors.setAnchor(scalar, anchor) return scalar as Scalar.Parsed } diff --git a/src/compose/compose-block-map.ts b/src/compose/resolve-block-map.ts similarity index 98% rename from src/compose/compose-block-map.ts rename to src/compose/resolve-block-map.ts index b057d9c7..2f947f1b 100644 --- a/src/compose/compose-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -4,7 +4,7 @@ import type { BlockMap } from '../parse/parser.js' import { composeNode } from './compose-node.js' import { resolveProps } from './resolve-props.js' -export function composeBlockMap( +export function resolveBlockMap( doc: Document.Parsed, { items, offset }: BlockMap, anchor: string | null, diff --git a/src/compose/compose-block-seq.ts b/src/compose/resolve-block-seq.ts similarity index 97% rename from src/compose/compose-block-seq.ts rename to src/compose/resolve-block-seq.ts index eea88feb..52828258 100644 --- a/src/compose/compose-block-seq.ts +++ b/src/compose/resolve-block-seq.ts @@ -4,7 +4,7 @@ import type { BlockSequence } from '../parse/parser.js' import { composeNode } from './compose-node.js' import { resolveProps } from './resolve-props.js' -export function composeBlockSeq( +export function resolveBlockSeq( doc: Document.Parsed, { items, offset }: BlockSequence, anchor: string | null, diff --git a/src/compose/compose-flow-collection.ts b/src/compose/resolve-flow-collection.ts similarity index 97% rename from src/compose/compose-flow-collection.ts rename to src/compose/resolve-flow-collection.ts index 14ab50c8..608dad72 100644 --- a/src/compose/compose-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -3,7 +3,7 @@ import type { Document } from '../doc/Document.js' import type { FlowCollection, SourceToken } from '../parse/parser.js' import { composeNode } from './compose-node.js' -export function composeFlowCollection( +export function resolveFlowCollection( doc: Document.Parsed, fc: FlowCollection, _anchor: string | null, @@ -24,7 +24,6 @@ export function composeFlowCollection( let anchor = '' let tagName = '' - // let atExplicitKey = false let atValueEnd = false function resetProps() { @@ -34,7 +33,6 @@ export function composeFlowCollection( newlines = '' anchor = '' tagName = '' - // atExplicitKey = false atValueEnd = false } @@ -97,7 +95,6 @@ export function composeFlowCollection( break } case 'explicit-key-ind': - // atExplicitKey = true if (anchor || tagName) onError(offset, 'Anchors and tags must be after the ? indicator') break diff --git a/src/constants.d.ts b/src/constants.d.ts index 380ba1e7..3f2977ab 100644 --- a/src/constants.d.ts +++ b/src/constants.d.ts @@ -27,3 +27,10 @@ export enum Type { SEQ = 'SEQ', SEQ_ITEM = 'SEQ_ITEM' } + +export const defaultTagPrefix : 'tag:yaml.org,2002:' +export const defaultTags : { + MAP: 'tag:yaml.org,2002:map', + SEQ: 'tag:yaml.org,2002:seq', + STR: 'tag:yaml.org,2002:str' +} diff --git a/src/tags/failsafe/map.js b/src/tags/failsafe/map.js index 2109c294..5a880a66 100644 --- a/src/tags/failsafe/map.js +++ b/src/tags/failsafe/map.js @@ -26,5 +26,8 @@ export const map = { default: true, nodeClass: YAMLMap, tag: 'tag:yaml.org,2002:map', - resolve: map => map + resolve(map, onError) { + if (!(map instanceof YAMLMap)) onError('Expected a mapping for this tag') + return map + } } diff --git a/src/tags/failsafe/seq.js b/src/tags/failsafe/seq.js index 9cb52148..8b79f153 100644 --- a/src/tags/failsafe/seq.js +++ b/src/tags/failsafe/seq.js @@ -22,5 +22,8 @@ export const seq = { default: true, nodeClass: YAMLSeq, tag: 'tag:yaml.org,2002:seq', - resolve: seq => seq + resolve(seq, onError) { + if (!(seq instanceof YAMLSeq)) onError('Expected a sequence for this tag') + return seq + } } From 88ca37511efb3e3ec9aa9c57f84df95b66f8bd0f Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Tue, 15 Dec 2020 02:21:52 +0200 Subject: [PATCH 35/89] Return null from parse() & parseDocument() for empty input --- src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/index.ts b/src/index.ts index 3664c5d9..deb76513 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,7 @@ export const parseAllDocuments = parseDocs export function parseDocument(src: string, options?: Options) { const docs = parseDocs(src, options) + if (docs.length === 0) return null const doc = docs[0] if ( docs.length > 1 && @@ -35,6 +36,7 @@ export function parse( } const doc = parseDocument(src, options) + if (!doc) return null doc.warnings.forEach(warning => warn(doc.options.logLevel, warning)) if (doc.errors.length > 0) { if (LogLevel.indexOf(doc.options.logLevel) >= LogLevel.ERROR) From a8fe00f2396b9be50659cba73f910c1188fed298 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Tue, 15 Dec 2020 02:22:13 +0200 Subject: [PATCH 36/89] Fix various bugs --- src/compose/resolve-block-map.ts | 2 ++ src/compose/resolve-block-scalar.ts | 6 +----- src/compose/resolve-block-seq.ts | 2 ++ src/compose/resolve-flow-collection.ts | 10 ++++++++-- src/compose/resolve-props.ts | 11 +++++++++-- src/parse/lexer.ts | 12 ++++++++++++ src/stringify/addComment.js | 8 +++++--- 7 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index 2f947f1b..ba84915b 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -1,4 +1,5 @@ import { Pair, YAMLMap } from '../ast/index.js' +import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' import type { BlockMap } from '../parse/parser.js' import { composeNode } from './compose-node.js' @@ -12,6 +13,7 @@ export function resolveBlockMap( ) { const start = offset const map = new YAMLMap(doc.schema) + map.type = Type.MAP if (anchor) doc.anchors.setAnchor(map, anchor) for (const { start, key, sep, value } of items) { diff --git a/src/compose/resolve-block-scalar.ts b/src/compose/resolve-block-scalar.ts index 8cf7d183..3ec7e0ce 100644 --- a/src/compose/resolve-block-scalar.ts +++ b/src/compose/resolve-block-scalar.ts @@ -24,11 +24,7 @@ export function resolveBlockScalar( // shortcut for empty contents if (!scalar.source || chompStart === 0) { const value = - header.chomp === '+' - ? lines.map(line => line[0]).join('\n') + '\n' - : header.chomp === '-' - ? '' - : '\n' + header.chomp === '+' ? lines.map(line => line[0]).join('\n') + '\n' : '' let length = header.length if (scalar.source) length += scalar.source.length return { value, type, comment: header.comment, length } diff --git a/src/compose/resolve-block-seq.ts b/src/compose/resolve-block-seq.ts index 52828258..ff78c783 100644 --- a/src/compose/resolve-block-seq.ts +++ b/src/compose/resolve-block-seq.ts @@ -1,4 +1,5 @@ import { YAMLSeq } from '../ast/index.js' +import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' import type { BlockSequence } from '../parse/parser.js' import { composeNode } from './compose-node.js' @@ -12,6 +13,7 @@ export function resolveBlockSeq( ) { const start = offset const seq = new YAMLSeq(doc.schema) + seq.type = Type.SEQ if (anchor) doc.anchors.setAnchor(seq, anchor) for (const { start, value } of items) { const props = resolveProps( diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index 608dad72..fa2da7a5 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -1,4 +1,5 @@ import { Node, Pair, YAMLMap, YAMLSeq } from '../ast/index.js' +import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' import type { FlowCollection, SourceToken } from '../parse/parser.js' import { composeNode } from './compose-node.js' @@ -12,6 +13,7 @@ export function resolveFlowCollection( let offset = fc.offset const isMap = fc.start.source === '{' const coll = isMap ? new YAMLMap(doc.schema) : new YAMLSeq(doc.schema) + coll.type = isMap ? Type.FLOW_MAP : Type.FLOW_SEQ if (_anchor) doc.anchors.setAnchor(coll, _anchor) let key: Node.Parsed | null = null @@ -24,6 +26,7 @@ export function resolveFlowCollection( let anchor = '' let tagName = '' + let atExplicitKey = false let atValueEnd = false function resetProps() { @@ -33,6 +36,7 @@ export function resolveFlowCollection( newlines = '' anchor = '' tagName = '' + atExplicitKey = false atValueEnd = false } @@ -43,7 +47,7 @@ export function resolveFlowCollection( const props = { spaceBefore, comment, anchor, tagName } value = composeNode(doc, offset, props, onError) } - if (isMap) { + if (isMap || atExplicitKey) { const pair = key ? new Pair(key, value) : new Pair(value) coll.items.push(pair) } else { @@ -97,8 +101,10 @@ export function resolveFlowCollection( case 'explicit-key-ind': if (anchor || tagName) onError(offset, 'Anchors and tags must be after the ? indicator') + atExplicitKey = true break case 'map-value-ind': { + atExplicitKey = false if (key) { if (value) { onError(offset, 'Missing {} around pair used as mapping key') @@ -139,7 +145,7 @@ export function resolveFlowCollection( } if (isSourceToken) offset += (token as SourceToken).source.length } - if (key || value) addItem() + if (key || value || atExplicitKey) addItem() coll.range = [fc.offset, offset] return coll as YAMLMap.Parsed | YAMLSeq.Parsed } diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index 0574016e..91a4bbb3 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -1,6 +1,13 @@ import { SourceToken } from '../parse/parser' import { StreamDirectives } from './stream-directives' +function isSpaceBefore(sep: string) { + if (!sep) return false + const first = sep.indexOf('\n') + if (first === -1) return false + return sep.includes('\n', first + 1) +} + export function resolveProps( directives: StreamDirectives, start: SourceToken[], @@ -27,7 +34,7 @@ export function resolveProps( case 'comment': { const cb = token.source.substring(1) if (!hasComment) { - if (sep) spaceBefore = true + if (isSpaceBefore(sep)) spaceBefore = true comment = cb } else comment += sep + cb hasComment = true @@ -57,6 +64,6 @@ export function resolveProps( } if (token.source) length += token.source.length } - if (!comment && sep) spaceBefore = true + if (!comment && isSpaceBefore(sep)) spaceBefore = true return { found, spaceBefore, comment, anchor, tagName, length } } diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index c27279ef..ad99fff7 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -101,6 +101,7 @@ export class Lexer { atEnd = false buffer = '' + flowKey = false flowLevel = 0 indent = 0 indentMore = '' @@ -280,6 +281,7 @@ export class Lexer { case '{': case '[': this.pushCount(1) + this.flowKey = false this.flowLevel = 1 return 'flow' case '}': @@ -317,17 +319,27 @@ export class Lexer { case '{': case '[': this.pushCount(1) + this.flowKey = false this.flowLevel += 1 return 'flow' case '}': case ']': this.pushCount(1) + this.flowKey = true this.flowLevel -= 1 return this.flowLevel ? 'flow' : 'doc' case '"': case "'": + this.flowKey = true return this.parseQuotedScalar() + case ':': + if (this.flowKey) { + this.pushCount(1) + this.pushSpaces() + return 'flow' + } + // fallthrough default: + this.flowKey = false return this.parsePlainScalar() } } diff --git a/src/stringify/addComment.js b/src/stringify/addComment.js index 52fe007c..5d221b90 100644 --- a/src/stringify/addComment.js +++ b/src/stringify/addComment.js @@ -7,7 +7,9 @@ export function addCommentBefore(str, indent, comment) { export function addComment(str, indent, comment) { return !comment ? str - : comment.indexOf('\n') === -1 - ? `${str} #${comment}` - : `${str}\n` + comment.replace(/^/gm, `${indent || ''}#`) + : comment.includes('\n') + ? `${str}\n` + comment.replace(/^/gm, `${indent || ''}#`) + : str.endsWith(' ') + ? `${str}#${comment}` + : `${str} #${comment}` } From 264c1c5084c2604166b8b8df7b8645f2674a00aa Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Tue, 15 Dec 2020 02:23:00 +0200 Subject: [PATCH 37/89] Adjust tests --- tests/doc/parse.js | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/doc/parse.js b/tests/doc/parse.js index 2ebdf02b..46dc8c66 100644 --- a/tests/doc/parse.js +++ b/tests/doc/parse.js @@ -75,6 +75,24 @@ describe('tags', () => { }) }) +describe('custom string on node', () => { + test('tiled null', () => { + YAML.scalarOptions.null.nullStr = '~' + const doc = YAML.parse('a: null') + const str = YAML.stringify(doc, { simpleKeys: true }) + expect(str).toBe('a: ~\n') + expect(YAML.parse(str)).toEqual({ a: null }) + }) + + test('empty string null', () => { + YAML.scalarOptions.null.nullStr = '' + const doc = YAML.parse('a: null') + const str = YAML.stringify(doc, { simpleKeys: true }) + expect(str).toBe('a: \n') + expect(YAML.parse(str)).toEqual({ a: null }) + }) +}) + describe('number types', () => { describe('asBigInt: false', () => { test('Version 1.1', () => { @@ -214,7 +232,7 @@ test('eemeli/yaml#3', () => { const src = '{ ? : 123 }' const doc = YAML.parseDocument(src) expect(doc.errors).toHaveLength(0) - expect(doc.contents.items[0].key).toBeNull() + expect(doc.contents.items[0].key.value).toBeNull() expect(doc.contents.items[0].value.value).toBe(123) }) @@ -352,13 +370,13 @@ describe('eemeli/yaml#l19', () => { test('map', () => { const src = 'a:\n # 123' const doc = YAML.parseDocument(src) - expect(String(doc)).toBe('? a\n\n# 123\n') + expect(String(doc)).toBe('a: # 123\n') }) test('seq', () => { const src = '- a: # 123' const doc = YAML.parseDocument(src) - expect(String(doc)).toBe('- ? a # 123\n') + expect(String(doc)).toBe('- a: # 123\n') }) }) @@ -422,17 +440,14 @@ test('comment between key & : in flow collection (eemeli/yaml#149)', () => { const src2 = '{a\n#c\n:1}' expect(() => YAML.parse(src2)).toThrow( - 'Indicator : missing in flow map entry' + 'Missing , between flow collection items' ) }) test('empty node should respect setOrigRanges()', () => { - const cst = YAML.parseCST('\r\na: # 123\r\n') - expect(cst).toHaveLength(1) - expect(cst.setOrigRanges()).toBe(true) - const doc = new YAML.Document(undefined, { keepCstNodes: true }).parse(cst[0]) - const empty = doc.contents.items[0].value.cstNode - expect(empty.range).toEqual({ start: 3, end: 3, origStart: 4, origEnd: 4 }) + const doc = YAML.parseDocument('\r\na: # 123\r\n') + const empty = doc.contents.items[0].value + expect(empty.range).toEqual([12, 12]) }) test('parse an empty string as null', () => { From d5d354279e5b510b252874964ca314fc76f6f615 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 20 Dec 2020 16:22:07 +0200 Subject: [PATCH 38/89] Fix some map parsing cases --- src/parse/parser.ts | 59 +++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 3f841ec2..538cd101 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -84,6 +84,11 @@ export type Token = | BlockSequence | FlowCollection +function includesToken(list: SourceToken[], type: SourceToken['type']) { + for (let i = 0; i < list.length; ++i) if (list[i].type === type) return true + return false +} + /** A YAML concrete syntax tree parser */ export class Parser { push: (token: Token) => void @@ -102,7 +107,10 @@ export class Parser { offset = 0 - /** Top indicates the bode that's currently being built */ + /** On the same line with a block map key */ + onKeyLine = false + + /** Top indicates the node that's currently being built */ stack: Token[] = [] /** The source of the current token, set in parse() */ @@ -236,7 +244,10 @@ export class Parser { const it = top.items[top.items.length - 1] if (it.value) top.items.push({ start: [], key: token, sep: [] }) else if (it.sep) it.value = token - else Object.assign(it, { key: token, sep: [] }) + else { + Object.assign(it, { key: token, sep: [] }) + this.onKeyLine = true + } break } case 'block-seq': { @@ -286,10 +297,10 @@ export class Parser { if (doc.value) return this.lineEnd(doc) switch (this.type) { case 'doc-start': { - const hasContent = doc.start.some( - ({ type }) => - type === 'doc-start' || type === 'anchor' || type === 'tag' - ) + const hasContent = + includesToken(doc.start, 'doc-start') || + includesToken(doc.start, 'anchor') || + includesToken(doc.start, 'tag') if (hasContent) { this.pop() this.step() @@ -330,6 +341,7 @@ export class Parser { indent: scalar.indent, items: [{ start: [], key: scalar, sep }] } + this.onKeyLine = true this.stack[this.stack.length - 1] = map } else this.lineEnd(scalar) } @@ -365,26 +377,30 @@ export class Parser { const it = map.items[map.items.length - 1] // it.sep is true-ish if pair already has key or : separator switch (this.type) { + case 'newline': + this.onKeyLine = false + // fallthrough case 'space': case 'comment': - case 'newline': if (it.value) map.items.push({ start: [this.sourceToken] }) else if (it.sep) it.sep.push(this.sourceToken) else it.start.push(this.sourceToken) return } if (this.indent >= map.indent) { + const atNextItem = !this.onKeyLine && this.indent === map.indent switch (this.type) { case 'anchor': case 'tag': - if (it.value) map.items.push({ start: [this.sourceToken] }) + if (atNextItem || it.value) + map.items.push({ start: [this.sourceToken] }) else if (it.sep) it.sep.push(this.sourceToken) else it.start.push(this.sourceToken) return case 'explicit-key-ind': if (!it.sep) it.start.push(this.sourceToken) - else if (it.value || this.indent === map.indent) + else if (atNextItem || it.value) map.items.push({ start: [this.sourceToken] }) else this.stack.push({ @@ -393,13 +409,17 @@ export class Parser { indent: this.indent, items: [{ start: [this.sourceToken] }] }) + this.onKeyLine = true return case 'map-value-ind': if (!it.sep) Object.assign(it, { key: null, sep: [this.sourceToken] }) - else if (it.value) + else if ( + it.value || + (atNextItem && !includesToken(it.start, 'explicit-key-ind')) + ) map.items.push({ start: [], key: null, sep: [this.sourceToken] }) - else if (it.sep.some(tok => tok.type === 'map-value-ind')) + else if (includesToken(it.sep, 'map-value-ind')) this.stack.push({ type: 'block-map', offset: this.offset, @@ -407,6 +427,7 @@ export class Parser { items: [{ start: [], key: null, sep: [this.sourceToken] }] }) else it.sep.push(this.sourceToken) + this.onKeyLine = true return case 'alias': @@ -414,9 +435,15 @@ export class Parser { case 'single-quoted-scalar': case 'double-quoted-scalar': { const fs = this.flowScalar(this.type) - if (it.value) map.items.push({ start: [], key: fs, sep: [] }) - else if (it.sep) this.stack.push(fs) - else Object.assign(it, { key: fs, sep: [] }) + if (atNextItem || it.value) { + map.items.push({ start: [], key: fs, sep: [] }) + this.onKeyLine = true + } else if (it.sep) { + this.stack.push(fs) + } else { + Object.assign(it, { key: fs, sep: [] }) + this.onKeyLine = true + } return } @@ -446,7 +473,7 @@ export class Parser { return case 'seq-item-ind': if (this.indent !== seq.indent) break - if (it.value || it.start.some(tok => tok.type === 'seq-item-ind')) + if (it.value || includesToken(it.start, 'seq-item-ind')) seq.items.push({ start: [this.sourceToken] }) else it.start.push(this.sourceToken) return @@ -540,6 +567,7 @@ export class Parser { items: [{ start: [this.sourceToken] }] } as BlockSequence case 'explicit-key-ind': + this.onKeyLine = true return { type: 'block-map', offset: this.offset, @@ -547,6 +575,7 @@ export class Parser { items: [{ start: [this.sourceToken] }] } as BlockMap case 'map-value-ind': + this.onKeyLine = true return { type: 'block-map', offset: this.offset, From f7a89a9d48103a6912582cef644c4dcaf2a42e69 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 20 Dec 2020 17:48:38 +0200 Subject: [PATCH 39/89] Allow flow collections to grab line-end comments & act as implicit map keys --- src/parse/parser.ts | 73 +++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 538cd101..e4b333fb 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -70,7 +70,7 @@ export interface FlowCollection { indent: number start: SourceToken items: Array - end?: SourceToken + end: SourceToken[] } export type Token = @@ -487,35 +487,49 @@ export class Parser { } flowCollection(fc: FlowCollection) { - switch (this.type) { - case 'space': - case 'comment': - case 'newline': - case 'comma': - case 'explicit-key-ind': - case 'map-value-ind': - case 'anchor': - case 'tag': - fc.items.push(this.sourceToken) - return + if (fc.end.length === 0) { + switch (this.type) { + case 'space': + case 'comment': + case 'newline': + case 'comma': + case 'explicit-key-ind': + case 'map-value-ind': + case 'anchor': + case 'tag': + fc.items.push(this.sourceToken) + return - case 'alias': - case 'scalar': - case 'single-quoted-scalar': - case 'double-quoted-scalar': - fc.items.push(this.flowScalar(this.type)) - return + case 'alias': + case 'scalar': + case 'single-quoted-scalar': + case 'double-quoted-scalar': + fc.items.push(this.flowScalar(this.type)) + return - case 'flow-map-end': - case 'flow-seq-end': - fc.end = this.sourceToken - this.pop() - return + case 'flow-map-end': + case 'flow-seq-end': + fc.end.push(this.sourceToken) + return + } + const bv = this.startBlockValue() + if (bv) return this.stack.push(bv) + this.pop() + this.step() + } else if (this.type === 'map-value-ind') { + const sep = fc.end.splice(1, fc.end.length) + sep.push(this.sourceToken) + const map: BlockMap = { + type: 'block-map', + offset: fc.offset, + indent: fc.indent, + items: [{ start: [], key: fc, sep }] + } + this.onKeyLine = true + this.stack[this.stack.length - 1] = map + } else { + this.lineEnd(fc) } - const bv = this.startBlockValue() - if (bv) return this.stack.push(bv) - this.pop() - this.step() } flowScalar( @@ -557,7 +571,8 @@ export class Parser { offset: this.offset, indent: this.indent, start: this.sourceToken, - items: [] + items: [], + end: [] } as FlowCollection case 'seq-item-ind': return { @@ -586,7 +601,7 @@ export class Parser { return null } - lineEnd(token: Document | FlowScalar) { + lineEnd(token: Document | FlowCollection | FlowScalar) { switch (this.type) { case 'space': case 'comment': From 51f9615abc383604c1deb74b0ccea0e79c36cba4 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 21 Dec 2020 02:54:19 +0200 Subject: [PATCH 40/89] Drop YAML 1.0 tests --- tests/doc/types.js | 56 ---------------------------------------------- 1 file changed, 56 deletions(-) diff --git a/tests/doc/types.js b/tests/doc/types.js index 6b6a0e38..0c8265e7 100644 --- a/tests/doc/types.js +++ b/tests/doc/types.js @@ -648,62 +648,6 @@ describe('custom tags', () => { ) }) - test('YAML 1.0 explicit tags', () => { - const src = `%YAML:1.0 ---- -date: 2001-01-23 -number: !int '123' -string: !str 123 -pool: !!ball { number: 8 } -perl: !perl/Text::Tabs {}` - - const doc = YAML.parseDocument(src) - expect(doc.version).toBe('1.0') - expect(doc.toJS()).toMatchObject({ - number: 123, - string: '123', - pool: { number: 8 }, - perl: {} - }) - const date = doc.contents.items[0].value.value - expect(date).toBeInstanceOf(Date) - expect(date.getFullYear()).toBe(2001) - expect(String(doc)).toBe(`%YAML:1.0 ---- -date: 2001-01-23 -number: !yaml.org,2002:int 123 -string: !yaml.org,2002:str "123" -pool: - !ball { number: 8 } -perl: - !perl/Text::Tabs {}\n`) - }) - - test('YAML 1.0 tag prefixing', () => { - const src = `%YAML:1.0 ---- -invoice: !domain.tld,2002/^invoice - customers: !seq - - !^customer - given : Chris - family : Dumars` - - const doc = YAML.parseDocument(src) - expect(doc.version).toBe('1.0') - expect(doc.toJS()).toMatchObject({ - invoice: { customers: [{ family: 'Dumars', given: 'Chris' }] } - }) - expect(String(doc)).toBe(`%YAML:1.0 ---- -invoice: - !domain.tld,2002/^invoice - customers: - !yaml.org,2002:seq - - !^customer - given: Chris - family: Dumars\n`) - }) - describe('custom tag objects', () => { const src = `!!binary | R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5 From 2401304820c5df860ee0328308076e80749c7888 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 28 Dec 2020 16:59:21 +0200 Subject: [PATCH 41/89] Resolve merge pairs --- src/ast/Merge.js | 6 +++--- src/ast/index.d.ts | 2 ++ src/compose/resolve-block-map.ts | 4 +++- src/compose/resolve-flow-collection.ts | 3 ++- src/compose/resolve-merge-pair.ts | 22 ++++++++++++++++++++++ src/resolve/resolveMap.js | 4 ++-- 6 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 src/compose/resolve-merge-pair.ts diff --git a/src/ast/Merge.js b/src/ast/Merge.js index 9c93e826..7b2279bf 100644 --- a/src/ast/Merge.js +++ b/src/ast/Merge.js @@ -3,9 +3,9 @@ import { Scalar } from './Scalar.js' import { YAMLMap } from './YAMLMap.js' import { YAMLSeq } from './YAMLSeq.js' -export const MERGE_KEY = '<<' - export class Merge extends Pair { + static KEY = '<<' + constructor(pair) { if (pair instanceof Pair) { let seq = pair.value @@ -17,7 +17,7 @@ export class Merge extends Pair { super(pair.key, seq) this.range = pair.range } else { - super(new Scalar(MERGE_KEY), new YAMLSeq()) + super(new Scalar(Merge.KEY), new YAMLSeq()) } this.type = Pair.Type.MERGE_PAIR } diff --git a/src/ast/index.d.ts b/src/ast/index.d.ts index 8ff9d8e0..8312e630 100644 --- a/src/ast/index.d.ts +++ b/src/ast/index.d.ts @@ -92,6 +92,8 @@ export namespace Pair { } export class Merge extends Pair { + static KEY: '<<' + constructor(pair?: Pair) type: Pair.Type.MERGE_PAIR /** Always Scalar('<<'), defined by the type specification */ key: AST.PlainValue diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index ba84915b..a23a1095 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -3,6 +3,7 @@ import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' import type { BlockMap } from '../parse/parser.js' import { composeNode } from './compose-node.js' +import { resolveMergePair } from './resolve-merge-pair.js' import { resolveProps } from './resolve-props.js' export function resolveBlockMap( @@ -60,7 +61,8 @@ export function resolveBlockMap( // value value const valueNode = composeNode(doc, value || offset, valueProps, onError) offset = valueNode.range[1] - map.items.push(new Pair(keyNode, valueNode)) + const pair = new Pair(keyNode, valueNode) + map.items.push(doc.schema.merge ? resolveMergePair(pair, onError) : pair) } else { // key with no value if (!keyProps.found) diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index fa2da7a5..d453ebd0 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -3,6 +3,7 @@ import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' import type { FlowCollection, SourceToken } from '../parse/parser.js' import { composeNode } from './compose-node.js' +import { resolveMergePair } from './resolve-merge-pair.js' export function resolveFlowCollection( doc: Document.Parsed, @@ -49,7 +50,7 @@ export function resolveFlowCollection( } if (isMap || atExplicitKey) { const pair = key ? new Pair(key, value) : new Pair(value) - coll.items.push(pair) + coll.items.push(doc.schema.merge ? resolveMergePair(pair, onError) : pair) } else { const seq = coll as YAMLSeq if (key) { diff --git a/src/compose/resolve-merge-pair.ts b/src/compose/resolve-merge-pair.ts new file mode 100644 index 00000000..cfd014fa --- /dev/null +++ b/src/compose/resolve-merge-pair.ts @@ -0,0 +1,22 @@ +import { Alias, Merge, Node, Pair, Scalar, YAMLMap } from '../ast/index.js' + +export function resolveMergePair( + pair: Pair, + onError: (offset: number, message: string) => void +) { + if (!(pair.key instanceof Scalar) || pair.key.value !== Merge.KEY) return pair + + const merge = new Merge(pair) + for (const node of merge.value.items as Node.Parsed[]) { + if (node instanceof Alias) { + if (node.source instanceof YAMLMap) { + // ok + } else { + onError(node.range[0], 'Merge nodes aliases can only point to maps') + } + } else { + onError(node.range[0], 'Merge nodes can only have Alias nodes as values') + } + } + return merge +} diff --git a/src/resolve/resolveMap.js b/src/resolve/resolveMap.js index 62195101..d3b622b9 100644 --- a/src/resolve/resolveMap.js +++ b/src/resolve/resolveMap.js @@ -1,5 +1,5 @@ import { Alias } from '../ast/Alias.js' -import { Merge, MERGE_KEY } from '../ast/Merge.js' +import { Merge } from '../ast/Merge.js' import { Pair } from '../ast/Pair.js' import { YAMLMap } from '../ast/YAMLMap.js' import { Char, Type } from '../constants.js' @@ -24,7 +24,7 @@ export function resolveMap(doc, cst) { resolveComments(map, comments) for (let i = 0; i < items.length; ++i) { const { key: iKey } = items[i] - if (doc.schema.merge && iKey && iKey.value === MERGE_KEY) { + if (doc.schema.merge && iKey && iKey.value === Merge.KEY) { items[i] = new Merge(items[i]) const sources = items[i].value.items let error = null From 4fce392cecf11e9339356f8b13ab1bcc51277949 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 28 Dec 2020 17:31:13 +0200 Subject: [PATCH 42/89] Fix alias parsing --- src/parse/lexer.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index ad99fff7..28221944 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -289,6 +289,9 @@ export class Lexer { // this is an error this.pushCount(1) return 'doc' + case '*': + this.pushUntil(isNotIdentifierChar) + return 'doc' case '"': case "'": return this.parseQuotedScalar() @@ -328,6 +331,9 @@ export class Lexer { this.flowKey = true this.flowLevel -= 1 return this.flowLevel ? 'flow' : 'doc' + case '*': + this.pushUntil(isNotIdentifierChar) + return 'flow' case '"': case "'": this.flowKey = true @@ -440,7 +446,6 @@ export class Lexer { switch (this.charAt(0)) { case '!': case '&': - case '*': return ( this.pushUntil(isNotIdentifierChar) + this.pushSpaces() + From 4ae69272323788ab4e10dc0006bf67424f19e20e Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 28 Dec 2020 18:23:41 +0200 Subject: [PATCH 43/89] Ensure Anchors#setAnchor() follows its documented behaviour --- docs/04_documents.md | 12 +++++++++--- src/doc/Anchors.js | 44 ++++++++++++++++++++++---------------------- tests/doc/anchors.js | 4 +++- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/docs/04_documents.md b/docs/04_documents.md index b561f2d9..b0ad88c5 100644 --- a/docs/04_documents.md +++ b/docs/04_documents.md @@ -162,7 +162,7 @@ A description of [alias and merge nodes](#alias-nodes) is included in the next s | getNames() | `string[]` | List of all defined anchor names. | | getNode(name: string) | `Node?` | The node associated with the anchor `name`, if set. | | newName(prefix: string) | `string` | Find an available anchor name with the given `prefix` and a numerical suffix. | -| setAnchor(node: Node, name?: string) | `string?` | Associate an anchor with `node`. If `name` is empty, a new name will be generated. | +| setAnchor(node?: Node, name?: string) | `string?` | Associate an anchor with `node`. If `name` is empty, a new name will be generated. | ```js const src = '[{ a: A }, { b: B }]' @@ -208,8 +208,14 @@ String(doc) // ] ``` -The constructors for `Alias` and `Merge` are not directly exported by the library, as they depend on the document's anchors; instead you'll need to use **`createAlias(node, name)`** and **`createMergePair(...sources)`**. You should make sure to only add alias and merge nodes to the document after the nodes to which they refer, or the document's YAML stringification will fail. +The constructors for `Alias` and `Merge` are not directly exported by the library, as they depend on the document's anchors; instead you'll need to use **`createAlias(node, name)`** and **`createMergePair(...sources)`**. +You should make sure to only add alias and merge nodes to the document after the nodes to which they refer, or the document's YAML stringification will fail. -It is valid to have an anchor associated with a node even if it has no aliases. `yaml` will not allow you to associate the same name with more than one node, even though this is allowed by the YAML spec (all but the last instance will have numerical suffixes added). To add or reassign an anchor, use **`setAnchor(node, name)`**. The second parameter is optional, and if left out either the pre-existing anchor name of the node will be used, or a new one generated. To remove an anchor, use `setAnchor(null, name)`. The function will return the new anchor's name, or `null` if both of its arguments are `null`. +It is valid to have an anchor associated with a node even if it has no aliases. +`yaml` will not allow you to associate the same name with more than one node, even though this is allowed by the YAML spec (all but the last instance will have numerical suffixes added). +To add or reassign an anchor, use **`setAnchor(node, name)`**. +The second parameter is optional, and if left out either the pre-existing anchor name of the node will be used, or a new one generated. +To remove an anchor, use `setAnchor(null, name)`. +The function will return the new anchor's name, or `null` if both of its arguments are `null`. While the `merge` option needs to be true to parse `Merge` nodes as such, this is not required during stringification. diff --git a/src/doc/Anchors.js b/src/doc/Anchors.js index 56f9b6a6..e7a4fc13 100644 --- a/src/doc/Anchors.js +++ b/src/doc/Anchors.js @@ -68,30 +68,30 @@ export class Anchors { } setAnchor(node, name) { - if (node != null && !Anchors.validAnchorNode(node)) { - throw new Error('Anchors may only be set for Scalar, Seq and Map nodes') - } - if (name && /[\x00-\x19\s,[\]{}]/.test(name)) { - throw new Error( - 'Anchor names must not contain whitespace or control characters' - ) - } const { map } = this - const prev = node && Object.keys(map).find(a => map[a] === node) - if (prev) { - if (!name) { - return prev - } else if (prev !== name) { - delete map[prev] - map[name] = node - } - } else { - if (!name) { - if (!node) return null - name = this.newName() - } - map[name] = node + if (!node) { + if (!name) return null + delete map[name] + return name } + + if (!Anchors.validAnchorNode(node)) + throw new Error('Anchors may only be set for Scalar, Seq and Map nodes') + if (name) { + if (/[\x00-\x19\s,[\]{}]/.test(name)) + throw new Error( + 'Anchor names must not contain whitespace or control characters' + ) + const prevNode = map[name] + if (prevNode && prevNode !== node) map[this.newName(name)] = prevNode + } + + const prevName = Object.keys(map).find(a => map[a] === node) + if (prevName) { + if (!name || prevName === name) return prevName + delete map[prevName] + } else if (!name) name = this.newName() + map[name] = node return name } } diff --git a/tests/doc/anchors.js b/tests/doc/anchors.js index 88d382a7..1bcc7031 100644 --- a/tests/doc/anchors.js +++ b/tests/doc/anchors.js @@ -59,7 +59,9 @@ describe('create', () => { expect(() => doc.anchors.setAnchor(a, 'A A')).toThrow( 'Anchor names must not contain whitespace or control characters' ) - expect(doc.anchors.getNames()).toMatchObject(['AA', 'a1', 'a2']) + expect(doc.anchors.setAnchor(a.items[0].value, 'AA')).toBe('AA') + expect(String(doc)).toBe('[ &AA1 { a: &AA A }, { b: &a2 B } ]\n') + expect(doc.anchors.getNames()).toMatchObject(['AA', 'a2', 'AA1']) }) test('doc.anchors.createAlias', () => { From f68fc4f9bf7f0112d024d32daa3af41b0647c41c Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 28 Dec 2020 19:19:27 +0200 Subject: [PATCH 44/89] Fix top-level block scalar parsing --- src/compose/resolve-block-scalar.ts | 1 + src/parse/lexer.ts | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/compose/resolve-block-scalar.ts b/src/compose/resolve-block-scalar.ts index 3ec7e0ce..730d3ef8 100644 --- a/src/compose/resolve-block-scalar.ts +++ b/src/compose/resolve-block-scalar.ts @@ -109,6 +109,7 @@ export function resolveBlockScalar( for (let i = chompStart; i < lines.length; ++i) value += '\n' + lines[i][0].slice(trimIndent) if (value[value.length - 1] !== '\n') value += '\n' + break default: value += '\n' } diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 28221944..9270bcda 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -384,13 +384,16 @@ export class Lexer { parseBlockScalar() { const reqIndent = this.indent > 0 ? this.indent + 1 : this.indentMore ? 1 : 0 - let nl = this.buffer.indexOf('\n', this.pos) + let nl = reqIndent === 0 ? -1 : this.buffer.indexOf('\n', this.pos) while (nl !== -1) { const cs = this.continueScalar(nl + 1, reqIndent) if (cs === -1) break nl = this.buffer.indexOf('\n', cs) } - if (nl === -1 && !this.atEnd) return this.setNext('block-scalar') + if (nl === -1) { + if (!this.atEnd) return this.setNext('block-scalar') + nl = this.buffer.length + } this.push(SCALAR) this.pushToIndex(nl + 1) return this.parseLineStart() From ec6835d44d06a0ec0db47647c42ab4e38c488b3f Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Wed, 30 Dec 2020 19:01:26 +0200 Subject: [PATCH 45/89] Lexer fixes: Empty scalars, tabs, cr-lf after plain, colon after plain in flow --- src/parse/lexer.ts | 75 ++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 9270bcda..c16cff5c 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -135,7 +135,7 @@ export class Lexer { atLineEnd() { let i = this.pos let ch = this.buffer[i] - while (ch === ' ') ch = this.buffer[++i] + while (ch === ' ' || ch === '\t') ch = this.buffer[++i] if (!ch || ch === '#' || ch === '\n') return true if (ch === '\r') return this.buffer[i + 1] === '\n' return false @@ -209,16 +209,24 @@ export class Lexer { const line = this.getLine() if (line === null) return this.setNext('stream') if (line[0] === '%') { - let dirEnd = line.indexOf(' #') + 1 - if (dirEnd === 0) dirEnd = line.length - while (line[dirEnd - 1] === ' ') dirEnd -= 1 - const n = this.pushCount(dirEnd) + this.pushSpaces() + let dirEnd = line.length + const cs = line.indexOf('#') + if (cs !== -1) { + const ch = line[cs - 1] + if (ch === ' ' || ch === '\t') dirEnd = cs - 1 + } + while (true) { + const ch = line[dirEnd - 1] + if (ch === ' ' || ch === '\t') dirEnd -= 1 + else break + } + const n = this.pushCount(dirEnd) + this.pushSpaces(true) this.pushCount(line.length - n) // possible comment this.pushNewline() return 'stream' } if (this.atLineEnd()) { - const sp = this.pushSpaces() + const sp = this.pushSpaces(true) this.pushCount(line.length - sp) this.pushNewline() return 'stream' @@ -242,7 +250,7 @@ export class Lexer { return 'stream' } } - this.indent = this.pushSpaces() + this.indent = this.pushSpaces(false) this.indentMore = '' return this.parseBlockStart() } @@ -252,7 +260,7 @@ export class Lexer { if (!ch1 && !this.atEnd) return this.setNext('block-start') if ((ch0 === '-' || ch0 === '?' || ch0 === ':') && isEmpty(ch1)) { const start = this.pos - const n = this.pushCount(1) + this.pushSpaces() + const n = this.pushCount(1) + this.pushSpaces(true) this.indentMore += this.buffer.substr(start, n) return this.parseBlockStart() } @@ -268,7 +276,7 @@ export class Lexer { } parseDocument() { - this.pushSpaces() + this.pushSpaces(true) const line = this.getLine() if (line === null) return this.setNext('doc') let n = this.pushIndicators() @@ -298,7 +306,7 @@ export class Lexer { case '|': case '>': n += this.pushUntil(isEmpty) - n += this.pushSpaces() + n += this.pushSpaces(true) this.pushCount(line.length - n) this.pushNewline() return this.parseBlockScalar() @@ -308,10 +316,10 @@ export class Lexer { } parseFlowCollection() { - while (this.pushNewline() + this.pushSpaces() > 0) {} + while (this.pushNewline() + this.pushSpaces(true) > 0) {} const line = this.getLine() if (line === null) return this.setNext('flow') - let n = line[0] === ',' ? this.pushCount(1) + this.pushSpaces() : 0 + let n = line[0] === ',' ? this.pushCount(1) + this.pushSpaces(true) : 0 n += this.pushIndicators() switch (line[n]) { case undefined: @@ -338,12 +346,15 @@ export class Lexer { case "'": this.flowKey = true return this.parseQuotedScalar() - case ':': - if (this.flowKey) { - this.pushCount(1) + this.pushSpaces() + case ':': { + const next = this.charAt(1) + if (this.flowKey || isEmpty(next) || next === ',') { + this.pushCount(1) + this.pushSpaces(true) return 'flow' } - // fallthrough + // fallthrough + } default: this.flowKey = false return this.parsePlainScalar() @@ -377,7 +388,7 @@ export class Lexer { if (nl !== -1 && nl < end) end = nl - 1 } if (end === -1) return this.setNext('quoted-scalar') - this.pushToIndex(end + 1) + this.pushToIndex(end + 1, false) return this.flowLevel ? 'flow' : 'doc' } @@ -395,7 +406,7 @@ export class Lexer { nl = this.buffer.length } this.push(SCALAR) - this.pushToIndex(nl + 1) + this.pushToIndex(nl + 1, true) return this.parseLineStart() } @@ -413,8 +424,9 @@ export class Lexer { const next = this.buffer[i + 1] if (next === '#' || (inFlow && invalidFlowScalarChars.includes(next))) break - if (ch === '\n') { - const cs = this.continueScalar(i + 1, reqIndent) + if (ch === '\n' || (ch === '\r' && next === '\n')) { + const ls = i + (ch === '\n' ? 1 : 2) + const cs = this.continueScalar(ls, reqIndent) if (cs === -1) break i = Math.max(i, cs - 2) // to advance, but still account for ' #' } @@ -422,7 +434,7 @@ export class Lexer { } if (!ch && !this.atEnd) return this.setNext('plain-scalar') this.push(SCALAR) - this.pushToIndex(i) + this.pushToIndex(i, true) return inFlow ? 'flow' : 'doc' } @@ -435,13 +447,13 @@ export class Lexer { return 0 } - pushToIndex(i: number) { + pushToIndex(i: number, allowEmpty: boolean) { const s = this.buffer.slice(this.pos, i) if (s) { this.push(s) this.pos += s.length return s.length - } + } else if (allowEmpty) this.push('') return 0 } @@ -451,7 +463,7 @@ export class Lexer { case '&': return ( this.pushUntil(isNotIdentifierChar) + - this.pushSpaces() + + this.pushSpaces(true) + this.pushIndicators() ) case ':': @@ -464,7 +476,9 @@ export class Lexer { } else { this.indentMore = ' ' } - return this.pushCount(1) + this.pushSpaces() + this.pushIndicators() + return ( + this.pushCount(1) + this.pushSpaces(true) + this.pushIndicators() + ) } } return 0 @@ -477,9 +491,12 @@ export class Lexer { else return 0 } - pushSpaces() { - let i = this.pos - while (this.buffer[i] === ' ') i += 1 + pushSpaces(allowTabs: boolean) { + let i = this.pos - 1 + let ch: string + do { + ch = this.buffer[++i] + } while (ch === ' ' || (allowTabs && ch === '\t')) const n = i - this.pos if (n > 0) { this.push(this.buffer.substr(this.pos, n)) @@ -492,6 +509,6 @@ export class Lexer { let i = this.pos let ch = this.buffer[i] while (!test(ch)) ch = this.buffer[++i] - return this.pushToIndex(i) + return this.pushToIndex(i, false) } } From 93975fa63cca253b6c251f32c2178e4671074e55 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Fri, 1 Jan 2021 11:51:39 +0200 Subject: [PATCH 46/89] Add strict option (default true), to allow being less pedantic --- docs/03_options.md | 1 + src/options.d.ts | 6 ++++++ src/options.js | 1 + 3 files changed, 8 insertions(+) diff --git a/docs/03_options.md b/docs/03_options.md index bd73f22e..1f2c5492 100644 --- a/docs/03_options.md +++ b/docs/03_options.md @@ -39,6 +39,7 @@ The `version` option value (`'1.2'` by default) may be overridden by any documen | schema | `'core' ⎮ 'failsafe' ⎮` `'json' ⎮ 'yaml-1.1'` | The base schema to use. By default `'core'` for YAML 1.2 and `'yaml-1.1'` for earlier versions. | | simpleKeys | `boolean` | When stringifying, require keys to be scalars and to use implicit rather than explicit notation. By default `false`. | | sortMapEntries | `boolean ⎮` `(a, b: Pair) => number` | When stringifying, sort map entries. If `true`, sort by comparing key values with `<`. By default `false`. | +| strict | `boolean` | When parsing, do not ignore errors required by the YAML 1.2 spec, but caused by unambiguous content. By default `true`. | | version | `'1.0' ⎮ '1.1' ⎮ '1.2'` | The YAML version used by documents without a `%YAML` directive. By default `'1.2'`. | [exponential entity expansion attacks]: https://en.wikipedia.org/wiki/Billion_laughs_attack diff --git a/src/options.d.ts b/src/options.d.ts index f76b4a2a..5138e29a 100644 --- a/src/options.d.ts +++ b/src/options.d.ts @@ -80,6 +80,12 @@ export interface Options extends Schema.Options { * Default: `false` */ simpleKeys?: boolean + /** + * When parsing, do not ignore errors required by the YAML 1.2 spec, but caused by unambiguous content. + * + * Default: `true` + */ + strict?: boolean /** * The YAML version used by documents without a `%YAML` directive. * diff --git a/src/options.js b/src/options.js index fcf0894f..16ac3ebb 100644 --- a/src/options.js +++ b/src/options.js @@ -20,6 +20,7 @@ export const defaultOptions = { maxAliasCount: 100, prettyErrors: true, simpleKeys: false, + strict: true, version: '1.2' } From 182a2de1dbba38a43150e2de794645c5d7a02897 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Fri, 1 Jan 2021 20:24:24 +0200 Subject: [PATCH 47/89] Loads of debugging for various tests --- src/compose/compose-doc.ts | 2 +- src/compose/parse-docs.ts | 8 ++- src/compose/resolve-block-map.ts | 18 ++++-- src/compose/resolve-block-scalar.ts | 3 +- src/compose/resolve-block-seq.ts | 2 +- src/compose/resolve-flow-collection.ts | 39 ++++++++---- src/compose/resolve-props.ts | 12 ++-- src/parse/lexer.ts | 75 ++++++++++++----------- src/parse/parser.ts | 82 ++++++++++++++++++++------ 9 files changed, 157 insertions(+), 84 deletions(-) diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index 847774ee..cffdf915 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -24,7 +24,7 @@ export function composeDoc( offset, onError ) - if (props.found) doc.directivesEndMarker = true + if (props.found !== -1) doc.directivesEndMarker = true doc.contents = composeNode( doc, diff --git a/src/compose/parse-docs.ts b/src/compose/parse-docs.ts index 0d2ddab1..5970bcb9 100644 --- a/src/compose/parse-docs.ts +++ b/src/compose/parse-docs.ts @@ -10,6 +10,7 @@ export function parseDocs(source: string, options?: Options) { const docs: Document.Parsed[] = [] const lines: number[] = [] + let atDirectives = false let comment = '' let errors: YAMLParseError[] = [] let warnings: YAMLWarning[] = [] @@ -35,11 +36,13 @@ export function parseDocs(source: string, options?: Options) { switch (token.type) { case 'directive': directives.add(token.source, onError) + atDirectives = true break case 'document': { const doc = composeDoc(options, directives, token, onError) decorate(doc) docs.push(doc) + atDirectives = false break } case 'comment': @@ -52,10 +55,13 @@ export function parseDocs(source: string, options?: Options) { const msg = token.source ? `${token.message}: ${JSON.stringify(token.source)}` : token.message - errors.push(new YAMLParseError(-1, msg)) + const error = new YAMLParseError(-1, msg) + if (atDirectives || docs.length === 0) errors.push(error) + else docs[docs.length - 1].errors.push(error) break } case 'space': + case 'doc-end': break default: console.log('###', token) diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index a23a1095..321c18de 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -6,9 +6,11 @@ import { composeNode } from './compose-node.js' import { resolveMergePair } from './resolve-merge-pair.js' import { resolveProps } from './resolve-props.js' +const startColMsg = 'All collection items must start at the same column' + export function resolveBlockMap( doc: Document.Parsed, - { items, offset }: BlockMap, + { indent, items, offset }: BlockMap, anchor: string | null, onError: (offset: number, message: string, warning?: boolean) => void ) { @@ -26,8 +28,10 @@ export function resolveBlockMap( offset, onError ) - if (!keyProps.found) { - // implicit key + const implicitKey = keyProps.found === -1 + if (implicitKey) { + if (key && 'indent' in key && key.indent !== indent) + onError(offset, startColMsg) if (keyProps.anchor || keyProps.tagName || sep) { // FIXME: check single-line // FIXME: check 1024 chars @@ -39,7 +43,7 @@ export function resolveBlockMap( } continue } - } + } else if (keyProps.found !== indent) onError(offset, startColMsg) offset += keyProps.length // key value @@ -57,7 +61,9 @@ export function resolveBlockMap( ) offset += valueProps.length - if (valueProps.found) { + if (valueProps.found !== -1) { + if (implicitKey && value?.type === 'block-map' && !valueProps.hasNewline) + onError(offset, 'Nested mappings are not allowed in compact mappings') // value value const valueNode = composeNode(doc, value || offset, valueProps, onError) offset = valueNode.range[1] @@ -65,7 +71,7 @@ export function resolveBlockMap( map.items.push(doc.schema.merge ? resolveMergePair(pair, onError) : pair) } else { // key with no value - if (!keyProps.found) + if (implicitKey) onError(keyStart, 'Implicit map keys need to be followed by map values') if (valueProps.comment) { if (map.comment) map.comment += '\n' + valueProps.comment diff --git a/src/compose/resolve-block-scalar.ts b/src/compose/resolve-block-scalar.ts index 730d3ef8..c26cca80 100644 --- a/src/compose/resolve-block-scalar.ts +++ b/src/compose/resolve-block-scalar.ts @@ -31,7 +31,8 @@ export function resolveBlockScalar( } // find the indentation level to trim from start - let trimIndent = scalar.indent + header.indent + // FIXME probably wrong for explicit indents + let trimIndent = header.indent let offset = scalar.offset + header.length let contentStart = 0 for (let i = 0; i < chompStart; ++i) { diff --git a/src/compose/resolve-block-seq.ts b/src/compose/resolve-block-seq.ts index ff78c783..b28c2415 100644 --- a/src/compose/resolve-block-seq.ts +++ b/src/compose/resolve-block-seq.ts @@ -24,7 +24,7 @@ export function resolveBlockSeq( onError ) offset += props.length - if (!props.found) { + if (props.found === -1) { if (props.anchor || props.tagName || value) { onError(offset, 'Sequence item without - indicator') } else { diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index d453ebd0..4fb531a5 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -29,24 +29,26 @@ export function resolveFlowCollection( let atExplicitKey = false let atValueEnd = false + let nlAfterValueInSeq = false + + function getProps() { + const props = { spaceBefore, comment, anchor, tagName } - function resetProps() { spaceBefore = false comment = '' hasComment = false newlines = '' anchor = '' tagName = '' - atExplicitKey = false - atValueEnd = false + + return props } function addItem() { if (value) { if (hasComment) value.comment = comment } else { - const props = { spaceBefore, comment, anchor, tagName } - value = composeNode(doc, offset, props, onError) + value = composeNode(doc, offset, getProps(), onError) } if (isMap || atExplicitKey) { const pair = key ? new Pair(key, value) : new Pair(value) @@ -59,7 +61,6 @@ export function resolveFlowCollection( seq.items.push(map) } else seq.items.push(value) } - resetProps() } for (const token of fc.items) { @@ -87,25 +88,30 @@ export function resolveFlowCollection( hasComment = false } atValueEnd = false - } else newlines += token.source + } else { + newlines += token.source + if (!isMap && !key && value) nlAfterValueInSeq = true + } break case 'anchor': if (anchor) onError(offset, 'A node can have at most one anchor') anchor = token.source.substring(1) + atValueEnd = false break case 'tag': { if (tagName) onError(offset, 'A node can have at most one tag') const tn = doc.directives.tagName(token.source, m => onError(offset, m)) if (tn) tagName = tn + atValueEnd = false break } case 'explicit-key-ind': if (anchor || tagName) onError(offset, 'Anchors and tags must be after the ? indicator') atExplicitKey = true + atValueEnd = false break case 'map-value-ind': { - atExplicitKey = false if (key) { if (value) { onError(offset, 'Missing {} around pair used as mapping key') @@ -116,32 +122,39 @@ export function resolveFlowCollection( value = null } // else explicit key } else if (value) { + if (doc.options.strict && nlAfterValueInSeq) + onError( + offset, + 'Implicit keys of flow sequence pairs need to be on a single line' + ) key = value value = null } else { - const props = { spaceBefore, comment, anchor, tagName } - key = composeNode(doc, offset, props, onError) // empty node - resetProps() + key = composeNode(doc, offset, getProps(), onError) // empty node } if (hasComment) { key.comment = comment comment = '' hasComment = false } + atExplicitKey = false + atValueEnd = false break } case 'comma': addItem() + atExplicitKey = false atValueEnd = true + nlAfterValueInSeq = false key = null value = null break default: { if (value) onError(offset, 'Missing , between flow collection items') - const props = { spaceBefore, comment, anchor, tagName } - value = composeNode(doc, token, props, onError) + value = composeNode(doc, token, getProps(), onError) offset = value.range[1] isSourceToken = false + atValueEnd = false } } if (isSourceToken) offset += (token as SourceToken).source.length diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index 91a4bbb3..7c93be27 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -10,7 +10,7 @@ function isSpaceBefore(sep: string) { export function resolveProps( directives: StreamDirectives, - start: SourceToken[], + tokens: SourceToken[], indicator: | 'doc-start' | 'explicit-key-ind' @@ -23,11 +23,12 @@ export function resolveProps( let spaceBefore = false let comment = '' let hasComment = false + let hasNewline = false let sep = '' let anchor = '' let tagName = '' - let found = false - for (const token of start) { + let found = -1 + for (const token of tokens) { switch (token.type) { case 'space': break @@ -42,6 +43,7 @@ export function resolveProps( break } case 'newline': + hasNewline = true sep += token.source break case 'anchor': @@ -57,7 +59,7 @@ export function resolveProps( } case indicator: // Could here handle preceding comments differently - found = true + found = token.indent break default: onError(offset + length, `Unexpected ${token.type} token`) @@ -65,5 +67,5 @@ export function resolveProps( if (token.source) length += token.source.length } if (!comment && isSpaceBefore(sep)) spaceBefore = true - return { found, spaceBefore, comment, anchor, tagName, length } + return { found, spaceBefore, comment, hasNewline, anchor, tagName, length } } diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index c16cff5c..ff025ac6 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -101,10 +101,10 @@ export class Lexer { atEnd = false buffer = '' - flowKey = false - flowLevel = 0 - indent = 0 - indentMore = '' + flowKey = false // can : immediately follow this flow node + flowLevel = 0 // count of surrounding flow collection levels + indentNext = 0 // minimum indent level for next line + indentValue = 0 // actual indent level of current line next: State | null = null pos = 0 @@ -145,14 +145,14 @@ export class Lexer { return this.buffer[this.pos + n] } - continueScalar(offset: number, reqIndent: number) { + continueScalar(offset: number) { let ch = this.buffer[offset] - if (reqIndent > 0) { + if (this.indentNext > 0) { let indent = 0 while (ch === ' ') ch = this.buffer[++indent + offset] if (ch === '\r' && this.buffer[indent + offset + 1] === '\n') return offset + indent + 1 - return ch === '\n' || indent >= reqIndent ? offset + indent : -1 + return ch === '\n' || indent >= this.indentNext ? offset + indent : -1 } if (ch === '-' || ch === '.') { const dt = this.buffer.substr(offset, 3) @@ -242,16 +242,16 @@ export class Lexer { const s = this.peek(3) if (s === '---' && isEmpty(this.charAt(3))) { this.pushCount(3) - this.indent = 0 - this.indentMore = '' + this.indentNext = 0 + this.indentValue = 0 return 'doc' } else if (s === '...' && isEmpty(this.charAt(3))) { this.pushCount(3) return 'stream' } } - this.indent = this.pushSpaces(false) - this.indentMore = '' + this.indentNext = this.pushSpaces(false) + this.indentValue = this.indentNext return this.parseBlockStart() } @@ -259,19 +259,12 @@ export class Lexer { const [ch0, ch1] = this.peek(2) if (!ch1 && !this.atEnd) return this.setNext('block-start') if ((ch0 === '-' || ch0 === '?' || ch0 === ':') && isEmpty(ch1)) { - const start = this.pos const n = this.pushCount(1) + this.pushSpaces(true) - this.indentMore += this.buffer.substr(start, n) + this.indentNext = this.indentValue + this.indentValue += n return this.parseBlockStart() } - if (this.indentMore.length > 2) { - let last = this.indentMore.length - 1 - while (this.indentMore[last] === ' ') last -= 1 - if (last > 0) { - this.indent += last - this.indentMore = this.indentMore.slice(last) - } - } + if (this.indentValue > this.indentNext) this.indentNext += 1 return 'doc' } @@ -378,14 +371,15 @@ export class Lexer { } let nl = this.buffer.indexOf('\n', this.pos) if (nl !== -1 && nl < end) { - const reqIndent = - this.indent > 0 ? this.indent + 1 : this.indentMore ? 1 : 0 while (nl !== -1 && nl < end) { - const cs = this.continueScalar(nl + 1, reqIndent) + const cs = this.continueScalar(nl + 1) if (cs === -1) break nl = this.buffer.indexOf('\n', cs) } - if (nl !== -1 && nl < end) end = nl - 1 + if (nl !== -1 && nl < end) { + // this is an error caused by an unexpected unindent + end = nl - 1 + } } if (end === -1) return this.setNext('quoted-scalar') this.pushToIndex(end + 1, false) @@ -393,11 +387,9 @@ export class Lexer { } parseBlockScalar() { - const reqIndent = - this.indent > 0 ? this.indent + 1 : this.indentMore ? 1 : 0 - let nl = reqIndent === 0 ? -1 : this.buffer.indexOf('\n', this.pos) + let nl = this.buffer.indexOf('\n', this.pos) while (nl !== -1) { - const cs = this.continueScalar(nl + 1, reqIndent) + const cs = this.continueScalar(nl + 1) if (cs === -1) break nl = this.buffer.indexOf('\n', cs) } @@ -412,8 +404,6 @@ export class Lexer { parsePlainScalar() { const inFlow = this.flowLevel > 0 - const reqIndent = - this.indent > 0 ? this.indent + 1 : this.indentMore ? 1 : 0 let i = this.pos - 1 let ch: string while ((ch = this.buffer[++i])) { @@ -426,7 +416,7 @@ export class Lexer { break if (ch === '\n' || (ch === '\r' && next === '\n')) { const ls = i + (ch === '\n' ? 1 : 2) - const cs = this.continueScalar(ls, reqIndent) + const cs = this.continueScalar(ls) if (cs === -1) break i = Math.max(i, cs - 2) // to advance, but still account for ' #' } @@ -460,6 +450,13 @@ export class Lexer { pushIndicators(): number { switch (this.charAt(0)) { case '!': + if (this.charAt(1) === '<') + return ( + this.pushVerbatimTag() + + this.pushSpaces(true) + + this.pushIndicators() + ) + // fallthrough case '&': return ( this.pushUntil(isNotIdentifierChar) + @@ -470,12 +467,7 @@ export class Lexer { case '?': // this is an error outside flow collections case '-': // this is an error if (isEmpty(this.charAt(1))) { - if (this.indentMore) { - this.indent += this.indentMore.length - this.indentMore = '' - } else { - this.indentMore = ' ' - } + this.indentNext = this.indentValue + 1 return ( this.pushCount(1) + this.pushSpaces(true) + this.pushIndicators() ) @@ -484,6 +476,13 @@ export class Lexer { return 0 } + pushVerbatimTag() { + let i = this.pos + 2 + let ch = this.buffer[i] + while (!isEmpty(ch) && ch !== '>') ch = this.buffer[++i] + return this.pushToIndex(ch === '>' ? i + 1 : i, false) + } + pushNewline() { const ch = this.buffer[this.pos] if (ch === '\n') return this.pushCount(1) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index e4b333fb..cb8d0831 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -89,6 +89,21 @@ function includesToken(list: SourceToken[], type: SourceToken['type']) { return false } +function isFlowToken( + token: Token | null +): token is FlowScalar | FlowCollection { + switch (token?.type) { + case 'alias': + case 'scalar': + case 'single-quoted-scalar': + case 'double-quoted-scalar': + case 'flow-collection': + return true + default: + return false + } +} + /** A YAML concrete syntax tree parser */ export class Parser { push: (token: Token) => void @@ -177,6 +192,8 @@ export class Parser { if (this.onNewLine) this.onNewLine(this.offset + source.length) break case 'space': + case 'explicit-key-ind': + case 'map-value-ind': case 'seq-item-ind': if (this.atNewLine) this.indent += source.length break @@ -315,8 +332,8 @@ export class Parser { doc.start.push(this.sourceToken) return case 'doc-end': - doc.start.push(this.sourceToken) this.pop() + this.push(this.sourceToken) return } const bv = this.startBlockValue() @@ -392,14 +409,16 @@ export class Parser { switch (this.type) { case 'anchor': case 'tag': - if (atNextItem || it.value) + if (atNextItem || it.value) { map.items.push({ start: [this.sourceToken] }) - else if (it.sep) it.sep.push(this.sourceToken) + this.onKeyLine = true + } else if (it.sep) it.sep.push(this.sourceToken) else it.start.push(this.sourceToken) return case 'explicit-key-ind': - if (!it.sep) it.start.push(this.sourceToken) + if (!it.sep && !includesToken(it.start, 'explicit-key-ind')) + it.start.push(this.sourceToken) else if (atNextItem || it.value) map.items.push({ start: [this.sourceToken] }) else @@ -426,7 +445,23 @@ export class Parser { indent: this.indent, items: [{ start: [], key: null, sep: [this.sourceToken] }] }) - else it.sep.push(this.sourceToken) + else if ( + includesToken(it.start, 'explicit-key-ind') && + isFlowToken(it.key) && + !includesToken(it.sep, 'newline') + ) { + const key = it.key + const sep = it.sep + sep.push(this.sourceToken) + // @ts-ignore type guard is wrong here + delete it.key, delete it.sep + this.stack.push({ + type: 'block-map', + offset: this.offset, + indent: this.indent, + items: [{ start: [], key, sep }] + }) + } else it.sep.push(this.sourceToken) this.onKeyLine = true return @@ -516,19 +551,28 @@ export class Parser { if (bv) return this.stack.push(bv) this.pop() this.step() - } else if (this.type === 'map-value-ind') { - const sep = fc.end.splice(1, fc.end.length) - sep.push(this.sourceToken) - const map: BlockMap = { - type: 'block-map', - offset: fc.offset, - indent: fc.indent, - items: [{ start: [], key: fc, sep }] - } - this.onKeyLine = true - this.stack[this.stack.length - 1] = map } else { - this.lineEnd(fc) + const parent = this.stack[this.stack.length - 2].type + if (parent === 'block-map') { + this.pop() + this.step() + } else if ( + this.type === 'map-value-ind' && + parent !== 'flow-collection' + ) { + const sep = fc.end.splice(1, fc.end.length) + sep.push(this.sourceToken) + const map: BlockMap = { + type: 'block-map', + offset: fc.offset, + indent: fc.indent, + items: [{ start: [], key: fc, sep }] + } + this.onKeyLine = true + this.stack[this.stack.length - 1] = map + } else { + this.lineEnd(fc) + } } } @@ -603,9 +647,11 @@ export class Parser { lineEnd(token: Document | FlowCollection | FlowScalar) { switch (this.type) { + case 'newline': + this.onKeyLine = false + // fallthrough case 'space': case 'comment': - case 'newline': if (token.end) token.end.push(this.sourceToken) else token.end = [this.sourceToken] if (this.type === 'newline') this.pop() From b3a97831bf6036b51e9b571a06222d9c6b304ec7 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Fri, 1 Jan 2021 23:09:59 +0200 Subject: [PATCH 48/89] Assign map & first-key props correctly --- src/parse/parser.ts | 57 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index cb8d0831..c959f798 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -104,6 +104,24 @@ function isFlowToken( } } +/** Note: May modify input array */ +function getFirstKeyStartProps(prev: SourceToken[]) { + if (prev.length === 0) return [] + + let i = prev.length + loop: while (--i >= 0) { + switch (prev[i].type) { + case 'explicit-key-ind': + case 'map-value-ind': + case 'seq-item-ind': + case 'newline': + break loop + } + } + while (prev[++i]?.type === 'space') {} + return prev.splice(i, prev.length) +} + /** A YAML concrete syntax tree parser */ export class Parser { push: (token: Token) => void @@ -215,7 +233,7 @@ export class Parser { } step() { - const top = this.peek() + const top = this.peek(1) if (!top) return this.stream() switch (top.type) { case 'document': @@ -237,8 +255,8 @@ export class Parser { this.pop() // error } - peek() { - return this.stack[this.stack.length - 1] + peek(n: number) { + return this.stack[this.stack.length - n] } pop(error?: Token) { @@ -249,7 +267,7 @@ export class Parser { } else if (this.stack.length === 0) { this.push(token) } else { - const top = this.peek() + const top = this.peek(1) switch (top.type) { case 'document': top.value = token @@ -346,17 +364,37 @@ export class Parser { scalar(scalar: FlowScalar) { if (this.type === 'map-value-ind') { + const parent = this.peek(2) + let prev: SourceToken[] + switch (parent.type) { + case 'document': + prev = parent.start + break + case 'block-map': { + const it = parent.items[parent.items.length - 1] + prev = it.sep || it.start + break + } + case 'block-seq': + prev = parent.items[parent.items.length - 1].start + break + default: + prev = [] + } + const start = getFirstKeyStartProps(prev) + let sep: SourceToken[] if (scalar.end) { sep = scalar.end sep.push(this.sourceToken) delete scalar.end } else sep = [this.sourceToken] + const map: BlockMap = { type: 'block-map', offset: scalar.offset, indent: scalar.indent, - items: [{ start: [], key: scalar, sep }] + items: [{ start, key: scalar, sep }] } this.onKeyLine = true this.stack[this.stack.length - 1] = map @@ -450,6 +488,7 @@ export class Parser { isFlowToken(it.key) && !includesToken(it.sep, 'newline') ) { + const start = getFirstKeyStartProps(it.start) const key = it.key const sep = it.sep sep.push(this.sourceToken) @@ -459,7 +498,7 @@ export class Parser { type: 'block-map', offset: this.offset, indent: this.indent, - items: [{ start: [], key, sep }] + items: [{ start, key, sep }] }) } else it.sep.push(this.sourceToken) this.onKeyLine = true @@ -552,13 +591,13 @@ export class Parser { this.pop() this.step() } else { - const parent = this.stack[this.stack.length - 2].type - if (parent === 'block-map') { + const parent = this.peek(2) + if (parent.type === 'block-map') { this.pop() this.step() } else if ( this.type === 'map-value-ind' && - parent !== 'flow-collection' + parent.type !== 'flow-collection' ) { const sep = fc.end.splice(1, fc.end.length) sep.push(this.sourceToken) From 0f84d865eb3d9f0a9d46bb0dc88f4717fe68ae50 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 2 Jan 2021 02:12:48 +0200 Subject: [PATCH 49/89] Include offset in error tokens where possible --- src/parse/parser.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index c959f798..d818fec6 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -9,6 +9,7 @@ export interface SourceToken { export interface ErrorToken { type: 'error' + offset?: number source: string message: string } @@ -194,7 +195,7 @@ export class Parser { const type = tokenType(source) if (!type) { const message = `Not a YAML token: ${source}` - this.pop({ type: 'error', source, message }) + this.pop({ type: 'error', offset: this.offset, message, source }) this.offset += source.length } else if (type === 'scalar') { this.atNewLine = false @@ -324,8 +325,12 @@ export class Parser { return } } - const message = `Unexpected ${this.type} token in YAML stream` - this.push({ type: 'error', message, source: this.source }) + this.push({ + type: 'error', + offset: this.offset, + message: `Unexpected ${this.type} token in YAML stream`, + source: this.source + }) } document(doc: Document) { @@ -357,8 +362,12 @@ export class Parser { const bv = this.startBlockValue() if (bv) this.stack.push(bv) else { - const message = `Unexpected ${this.type} token in YAML document` - this.push({ type: 'error', message, source: this.source }) + this.push({ + type: 'error', + offset: this.offset, + message: `Unexpected ${this.type} token in YAML document`, + source: this.source + }) } } From 3f0d5ec1b04d63052fb995bb9f49fcddfca38846 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 2 Jan 2021 02:11:49 +0200 Subject: [PATCH 50/89] Fix block-scalar indentation control --- src/compose/resolve-block-scalar.ts | 57 +++++++++++++---------------- src/compose/resolve-flow-scalar.ts | 4 +- src/parse/lexer.ts | 57 +++++++++++++++++++++++------ src/parse/parser.ts | 4 ++ 4 files changed, 77 insertions(+), 45 deletions(-) diff --git a/src/compose/resolve-block-scalar.ts b/src/compose/resolve-block-scalar.ts index c26cca80..9e5f60df 100644 --- a/src/compose/resolve-block-scalar.ts +++ b/src/compose/resolve-block-scalar.ts @@ -31,8 +31,7 @@ export function resolveBlockScalar( } // find the indentation level to trim from start - // FIXME probably wrong for explicit indents - let trimIndent = header.indent + let trimIndent = scalar.indent + header.indent let offset = scalar.offset + header.length let contentStart = 0 for (let i = 0; i < chompStart; ++i) { @@ -46,7 +45,7 @@ export function resolveBlockScalar( 'Block scalars with more-indented leading empty lines must use an explicit indentation indicator' onError(offset + indent.length, message) } - trimIndent = indent.length + if (header.indent === 0) trimIndent = indent.length contentStart = i break } @@ -67,39 +66,33 @@ export function resolveBlockScalar( const crlf = content[content.length - 1] === '\r' if (crlf) content = content.slice(0, -1) - if (indent.length < trimIndent) { - if (content === '') { - // empty line - if (sep === '\n') value += '\n' - else sep = '\n' - continue - } else { - const src = header.indent - ? 'explicit indentation indicator' - : 'first line' - const message = `Block scalar lines must not be less indented than their ${src}` - onError(offset - content.length - (crlf ? 2 : 1), message) - indent = '' - } + if (content && indent.length < trimIndent) { + const src = header.indent + ? 'explicit indentation indicator' + : 'first line' + const message = `Block scalar lines must not be less indented than their ${src}` + onError(offset - content.length - (crlf ? 2 : 1), message) + indent = '' } - if (type === Type.BLOCK_FOLDED) { - if (!indent || indent.length === trimIndent) { - value += sep + content - sep = ' ' - prevMoreIndented = false - } else { - // more-indented content within a folded block - if (sep === ' ') sep = '\n' - else if (!prevMoreIndented && sep === '\n') sep = '\n\n' - value += sep + indent.slice(trimIndent) + content - sep = '\n' - prevMoreIndented = true - } - } else { - // literal + if (type === Type.BLOCK_LITERAL) { value += sep + indent.slice(trimIndent) + content sep = '\n' + } else if (indent.length > trimIndent || content[0] === '\t') { + // more-indented content within a folded block + if (sep === ' ') sep = '\n' + else if (!prevMoreIndented && sep === '\n') sep = '\n\n' + value += sep + indent.slice(trimIndent) + content + sep = '\n' + prevMoreIndented = true + } else if (content === '') { + // empty line + if (sep === '\n') value += '\n' + else sep = '\n' + } else { + value += sep + content + sep = ' ' + prevMoreIndented = false } } diff --git a/src/compose/resolve-flow-scalar.ts b/src/compose/resolve-flow-scalar.ts index a2b6776d..b194c501 100644 --- a/src/compose/resolve-flow-scalar.ts +++ b/src/compose/resolve-flow-scalar.ts @@ -66,7 +66,7 @@ function singleQuotedValue( source: string, onError: (relOffset: number, message: string) => void ) { - if (source[source.length - 1] !== "'") + if (source[source.length - 1] !== "'" || source.length === 1) onError(source.length, "Missing closing 'quote") return foldLines(source.slice(1, -1)).replace(/''/g, "'") } @@ -126,7 +126,7 @@ function doubleQuotedValue( res += ch } } - if (source[source.length - 1] !== '"') + if (source[source.length - 1] !== '"' || source.length === 1) onError(source.length, 'Missing closing "quote') return res } diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index ff025ac6..1c62e664 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -101,6 +101,7 @@ export class Lexer { atEnd = false buffer = '' + blockScalarIndent = -1 // non-negative if explicitly in header flowKey = false // can : immediately follow this flow node flowLevel = 0 // count of surrounding flow collection levels indentNext = 0 // minimum indent level for next line @@ -298,7 +299,7 @@ export class Lexer { return this.parseQuotedScalar() case '|': case '>': - n += this.pushUntil(isEmpty) + n += this.parseBlockScalarHeader() n += this.pushSpaces(true) this.pushCount(line.length - n) this.pushNewline() @@ -386,19 +387,53 @@ export class Lexer { return this.flowLevel ? 'flow' : 'doc' } + parseBlockScalarHeader() { + let i = this.pos + while (true) { + const ch = this.buffer[++i] + if (ch === '-' || ch === '+') continue + const n = Number(ch) + this.blockScalarIndent = n > 0 ? n - 1 : -1 + break + } + return this.pushUntil(isEmpty) + } + parseBlockScalar() { - let nl = this.buffer.indexOf('\n', this.pos) - while (nl !== -1) { - const cs = this.continueScalar(nl + 1) - if (cs === -1) break - nl = this.buffer.indexOf('\n', cs) + let nl = this.pos - 1 + let indent = 0 + let ch: string + loop: for (let i = this.pos; (ch = this.buffer[i]); ++i) { + switch (ch) { + case ' ': + indent += 1 + break + case '\n': + nl = i + indent = 0 + break + default: + break loop + } } - if (nl === -1) { - if (!this.atEnd) return this.setNext('block-scalar') - nl = this.buffer.length + if (ch && indent < this.indentNext) { + this.push(SCALAR) + this.push('') + } else { + if (this.blockScalarIndent === -1) this.indentNext = indent + else this.indentNext += this.blockScalarIndent + while (nl !== -1) { + const cs = this.continueScalar(nl + 1) + if (cs === -1) break + nl = this.buffer.indexOf('\n', cs) + } + if (nl === -1) { + if (!this.atEnd) return this.setNext('block-scalar') + nl = this.buffer.length + } + this.push(SCALAR) + this.pushToIndex(nl + 1, true) } - this.push(SCALAR) - this.pushToIndex(nl + 1, true) return this.parseLineStart() } diff --git a/src/parse/parser.ts b/src/parse/parser.ts index d818fec6..034dad7b 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -214,6 +214,7 @@ export class Parser { case 'explicit-key-ind': case 'map-value-ind': case 'seq-item-ind': + // TODO: also track parent indent if (this.atNewLine) this.indent += source.length break case 'doc-mode': @@ -269,6 +270,9 @@ export class Parser { this.push(token) } else { const top = this.peek(1) + // For these, parent indent is needed instead of own + if (token.type === 'block-scalar' || token.type === 'flow-collection') + token.indent = 'indent' in top ? top.indent : -1 switch (top.type) { case 'document': top.value = token From 26c3a52814bd81a365734a1faba040159d8a34d1 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 2 Jan 2021 14:03:54 +0200 Subject: [PATCH 51/89] Require space before comment --- src/compose/compose-doc.ts | 14 +++----- src/compose/compose-node.ts | 18 ++++++---- src/compose/compose-scalar.ts | 4 +-- src/compose/resolve-block-map.ts | 6 ++-- src/compose/resolve-block-scalar.ts | 12 ++++++- src/compose/resolve-block-seq.ts | 3 +- src/compose/resolve-end.ts | 46 +++++++++++++++++++------- src/compose/resolve-flow-collection.ts | 33 ++++++++++++++++-- src/compose/resolve-flow-scalar.ts | 10 ++++-- src/compose/resolve-props.ts | 21 ++++++++++-- src/parse/lexer.ts | 2 +- 11 files changed, 127 insertions(+), 42 deletions(-) diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index cffdf915..529ee5bc 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -17,13 +17,7 @@ export function composeDoc( doc.version = directives.yaml.version doc.setSchema() // FIXME: always do this in the constructor - const props = resolveProps( - doc.directives, - start, - 'doc-start', - offset, - onError - ) + const props = resolveProps(doc, start, true, 'doc-start', offset, onError) if (props.found !== -1) doc.directivesEndMarker = true doc.contents = composeNode( @@ -32,9 +26,9 @@ export function composeDoc( props, onError ) - const { comment, length } = resolveEnd(end) - if (comment) doc.comment = comment - doc.range = [offset, doc.contents.range[1] + length] + const re = resolveEnd(end, doc.contents.range[1], false, onError) + if (re.comment) doc.comment = re.comment + doc.range = [offset, re.offset] return doc } diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts index b7edfa10..0acbf62e 100644 --- a/src/compose/compose-node.ts +++ b/src/compose/compose-node.ts @@ -24,7 +24,7 @@ export function composeNode( let node: Node.Parsed switch (token.type) { case 'alias': - node = composeAlias(doc.anchors, token, onError) + node = composeAlias(doc, token, onError) if (anchor || tagName) onError(token.offset, 'An alias node must not specify any properties') break @@ -65,16 +65,22 @@ function composeEmptyNode( } function composeAlias( - anchors: Document.Anchors, + doc: Document.Parsed, { offset, source, end }: FlowScalar, onError: (offset: number, message: string, warning?: boolean) => void ) { const name = source.substring(1) - const src = anchors.getNode(name) + const src = doc.anchors.getNode(name) if (!src) onError(offset, `Aliased anchor not found: ${name}`) const alias = new Alias(src as Node) - const { comment, length } = resolveEnd(end) - alias.range = [offset, offset + source.length + length] - if (comment) alias.comment = comment + + const re = resolveEnd( + end, + offset + source.length, + doc.options.strict, + onError + ) + alias.range = [offset, re.offset] + if (re.comment) alias.comment = re.comment return alias as Alias.Parsed } diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index d1fae1b8..b361fb96 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -15,8 +15,8 @@ export function composeScalar( const { offset } = token const { value, type, comment, length } = token.type === 'block-scalar' - ? resolveBlockScalar(token, onError) - : resolveFlowScalar(token, onError) + ? resolveBlockScalar(token, doc.options.strict, onError) + : resolveFlowScalar(token, doc.options.strict, onError) const tag = findScalarTagByName(doc.schema, value, tagName, onError) || diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index 321c18de..fe7a488a 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -22,8 +22,9 @@ export function resolveBlockMap( for (const { start, key, sep, value } of items) { // key properties const keyProps = resolveProps( - doc.directives, + doc, start, + true, 'explicit-key-ind', offset, onError @@ -53,8 +54,9 @@ export function resolveBlockMap( // value properties const valueProps = resolveProps( - doc.directives, + doc, sep || [], + !key || key.type === 'block-scalar', 'map-value-ind', offset, onError diff --git a/src/compose/resolve-block-scalar.ts b/src/compose/resolve-block-scalar.ts index 9e5f60df..b82d3fed 100644 --- a/src/compose/resolve-block-scalar.ts +++ b/src/compose/resolve-block-scalar.ts @@ -3,6 +3,7 @@ import type { BlockScalar } from '../parse/parser.js' export function resolveBlockScalar( scalar: BlockScalar, + strict: boolean, onError: (offset: number, message: string) => void ): { value: string @@ -10,7 +11,7 @@ export function resolveBlockScalar( comment: string length: number } { - const header = parseBlockScalarHeader(scalar, onError) + const header = parseBlockScalarHeader(scalar, strict, onError) if (!header) return { value: '', type: null, comment: '', length: 0 } const type = header.mode === '>' ? Type.BLOCK_FOLDED : Type.BLOCK_LITERAL const lines = scalar.source ? splitLines(scalar.source) : [] @@ -118,6 +119,7 @@ export function resolveBlockScalar( function parseBlockScalarHeader( { offset, props }: BlockScalar, + strict: boolean, onError: (offset: number, message: string) => void ) { if (props[0].type !== 'block-scalar-header') { @@ -140,16 +142,24 @@ function parseBlockScalarHeader( } if (error !== -1) onError(error, `Block scalar header includes extra characters: ${source}`) + let hasSpace = false let comment = '' let length = source.length for (let i = 1; i < props.length; ++i) { const token = props[i] switch (token.type) { case 'space': + hasSpace = true + // fallthrough case 'newline': length += token.source.length break case 'comment': + if (strict && !hasSpace) { + const message = + 'Comments must be separated from other tokens by white space characters' + onError(offset + length, message) + } length += token.source.length comment = token.source.substring(1) break diff --git a/src/compose/resolve-block-seq.ts b/src/compose/resolve-block-seq.ts index b28c2415..1bfc6b14 100644 --- a/src/compose/resolve-block-seq.ts +++ b/src/compose/resolve-block-seq.ts @@ -17,8 +17,9 @@ export function resolveBlockSeq( if (anchor) doc.anchors.setAnchor(seq, anchor) for (const { start, value } of items) { const props = resolveProps( - doc.directives, + doc, start, + true, 'seq-item-ind', offset, onError diff --git a/src/compose/resolve-end.ts b/src/compose/resolve-end.ts index 43b244b1..3e661f1d 100644 --- a/src/compose/resolve-end.ts +++ b/src/compose/resolve-end.ts @@ -1,21 +1,43 @@ import { SourceToken } from '../parse/parser.js' -export function resolveEnd(end: SourceToken[] | undefined) { +export function resolveEnd( + end: SourceToken[] | undefined, + offset: number, + reqSpace: boolean, + onError: (offset: number, message: string) => void +) { let comment = '' - let length = 0 if (end) { + let hasSpace = false let hasComment = false let sep = '' - for (const token of end) { - if (token.type === 'comment') { - const cb = token.source.substring(1) - if (!hasComment) comment = cb - else comment += sep + cb - hasComment = true - sep = '' - } else if (hasComment && token.type === 'newline') sep += token.source - length += token.source.length + for (const { source, type } of end) { + switch (type) { + case 'space': + hasSpace = true + break + case 'comment': { + if (reqSpace && !hasSpace) + onError( + offset, + 'Comments must be separated from other tokens by white space characters' + ) + const cb = source.substring(1) + if (!hasComment) comment = cb + else comment += sep + cb + hasComment = true + sep = '' + break + } + case 'newline': + if (hasComment) sep += source + hasSpace = true + break + default: + onError(offset, `Unexpected ${type} at node end`) + } + offset += source.length } } - return { comment, length } + return { comment, offset } } diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index 4fb531a5..3d79c5ac 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -3,6 +3,7 @@ import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' import type { FlowCollection, SourceToken } from '../parse/parser.js' import { composeNode } from './compose-node.js' +import { resolveEnd } from './resolve-end.js' import { resolveMergePair } from './resolve-merge-pair.js' export function resolveFlowCollection( @@ -22,6 +23,7 @@ export function resolveFlowCollection( let spaceBefore = false let comment = '' + let hasSpace = false let hasComment = false let newlines = '' let anchor = '' @@ -67,8 +69,14 @@ export function resolveFlowCollection( let isSourceToken = true switch (token.type) { case 'space': + hasSpace = true break case 'comment': + if (doc.options.strict && !hasSpace) + onError( + offset, + 'Comments must be separated from other tokens by white space characters' + ) const cb = token.source.substring(1) if (!hasComment) { if (newlines) spaceBefore = true @@ -92,17 +100,20 @@ export function resolveFlowCollection( newlines += token.source if (!isMap && !key && value) nlAfterValueInSeq = true } + hasSpace = true break case 'anchor': if (anchor) onError(offset, 'A node can have at most one anchor') anchor = token.source.substring(1) atValueEnd = false + hasSpace = false break case 'tag': { if (tagName) onError(offset, 'A node can have at most one tag') const tn = doc.directives.tagName(token.source, m => onError(offset, m)) if (tn) tagName = tn atValueEnd = false + hasSpace = false break } case 'explicit-key-ind': @@ -110,6 +121,7 @@ export function resolveFlowCollection( onError(offset, 'Anchors and tags must be after the ? indicator') atExplicitKey = true atValueEnd = false + hasSpace = false break case 'map-value-ind': { if (key) { @@ -139,15 +151,17 @@ export function resolveFlowCollection( } atExplicitKey = false atValueEnd = false + hasSpace = false break } case 'comma': addItem() + key = null + value = null atExplicitKey = false atValueEnd = true + hasSpace = false nlAfterValueInSeq = false - key = null - value = null break default: { if (value) onError(offset, 'Missing , between flow collection items') @@ -155,11 +169,26 @@ export function resolveFlowCollection( offset = value.range[1] isSourceToken = false atValueEnd = false + hasSpace = false } } if (isSourceToken) offset += (token as SourceToken).source.length } if (key || value || atExplicitKey) addItem() + + const expectedEnd = isMap ? '}' : ']' + const [ce, ...ee] = fc.end + if (!ce || ce.source !== expectedEnd) { + const cs = isMap ? 'map' : 'sequence' + onError(offset, `Expected flow ${cs} to end with ${expectedEnd}`) + } + if (ce) offset += ce.source.length + if (ee.length > 0) { + const end = resolveEnd(ee, offset, doc.options.strict, onError) + if (end.comment) coll.comment = comment + offset = end.offset + } + coll.range = [fc.offset, offset] return coll as YAMLMap.Parsed | YAMLSeq.Parsed } diff --git a/src/compose/resolve-flow-scalar.ts b/src/compose/resolve-flow-scalar.ts index b194c501..7aa1871a 100644 --- a/src/compose/resolve-flow-scalar.ts +++ b/src/compose/resolve-flow-scalar.ts @@ -4,6 +4,7 @@ import { resolveEnd } from './resolve-end.js' export function resolveFlowScalar( { offset, type, source, end }: FlowScalar, + strict: boolean, onError: (offset: number, message: string) => void ): { value: string @@ -40,8 +41,13 @@ export function resolveFlowScalar( } } - const { comment, length } = resolveEnd(end) - return { value, type: _type, comment, length: source.length + length } + const re = resolveEnd(end, 0, strict, _onError) + return { + value, + type: _type, + comment: re.comment, + length: source.length + re.offset + } } function plainValue( diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index 7c93be27..88878f3e 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -1,5 +1,5 @@ +import type { Document } from '../doc/Document.js' import { SourceToken } from '../parse/parser' -import { StreamDirectives } from './stream-directives' function isSpaceBefore(sep: string) { if (!sep) return false @@ -9,8 +9,9 @@ function isSpaceBefore(sep: string) { } export function resolveProps( - directives: StreamDirectives, + doc: Document.Parsed, tokens: SourceToken[], + startOnNewline: boolean, indicator: | 'doc-start' | 'explicit-key-ind' @@ -21,6 +22,7 @@ export function resolveProps( ) { let length = 0 let spaceBefore = false + let hasSpace = startOnNewline let comment = '' let hasComment = false let hasNewline = false @@ -31,8 +33,14 @@ export function resolveProps( for (const token of tokens) { switch (token.type) { case 'space': + hasSpace = true break case 'comment': { + if (doc.options.strict && !hasSpace) + onError( + offset + length, + 'Comments must be separated from other tokens by white space characters' + ) const cb = token.source.substring(1) if (!hasComment) { if (isSpaceBefore(sep)) spaceBefore = true @@ -44,25 +52,32 @@ export function resolveProps( } case 'newline': hasNewline = true + hasSpace = true sep += token.source break case 'anchor': if (anchor) onError(offset + length, 'A node can have at most one anchor') anchor = token.source.substring(1) + hasSpace = false break case 'tag': { if (tagName) onError(offset + length, 'A node can have at most one tag') - const tn = directives.tagName(token.source, msg => onError(offset, msg)) + const tn = doc.directives.tagName(token.source, msg => + onError(offset, msg) + ) if (tn) tagName = tn + hasSpace = false break } case indicator: // Could here handle preceding comments differently found = token.indent + hasSpace = false break default: onError(offset + length, `Unexpected ${token.type} token`) + hasSpace = false } if (token.source) length += token.source.length } diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 1c62e664..fa029bc4 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -396,7 +396,7 @@ export class Lexer { this.blockScalarIndent = n > 0 ? n - 1 : -1 break } - return this.pushUntil(isEmpty) + return this.pushUntil(ch => isEmpty(ch) || ch === '#') } parseBlockScalar() { From 147224406118a92a777a38a380847c7451f5f7be Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 2 Jan 2021 14:29:49 +0200 Subject: [PATCH 52/89] Allow skipping some yaml-test-suite tests --- tests/yaml-test-suite.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/yaml-test-suite.js b/tests/yaml-test-suite.js index ff850797..05754eea 100644 --- a/tests/yaml-test-suite.js +++ b/tests/yaml-test-suite.js @@ -4,6 +4,10 @@ import path from 'path' import * as YAML from '../index.js' import { testEvents } from '../dist/test-events.js' +const skip = { + SF5V: ['errors'] // not erroring for duplicate %YAML directives +} + const testDirs = fs .readdirSync(path.resolve(__dirname, 'yaml-test-suite')) .filter(dir => /^[A-Z0-9]{4}$/.test(dir)) @@ -67,17 +71,23 @@ testDirs.forEach(dir => { /* ignore error */ } + function _test(name, cb) { + const sd = skip[dir] + if (sd === true || (sd && sd.includes(name))) test.skip(name, cb) + else test(name, cb) + } + describe(`${dir}: ${name}`, () => { const docs = YAML.parseAllDocuments(yaml, { resolveKnownTags: false }) if (events) { - test('test.event', () => { + _test('test.event', () => { const res = testEvents(yaml) expect(res.events.join('\n') + '\n').toBe(events) expect(res.error).toBeNull() }) } - if (json) test('in.json', () => matchJson(docs, json)) - test('errors', () => { + if (json) _test('in.json', () => matchJson(docs, json)) + _test('errors', () => { const errors = docs .map(doc => doc.errors) .filter(docErrors => docErrors.length > 0) @@ -98,10 +108,10 @@ testDirs.forEach(dir => { '\nOUT-JSON\n' + JSON.stringify(src2), '\nRE-JSON\n' + JSON.stringify(docs2[0], null, ' ') - if (json) test('stringfy+re-parse', () => matchJson(docs2, json)) + if (json) _test('stringfy+re-parse', () => matchJson(docs2, json)) if (outYaml) { - test('out.yaml', () => { + _test('out.yaml', () => { const resDocs = YAML.parseAllDocuments(yaml, { mapAsMap: true }) const resJson = resDocs.map(doc => doc.toJS()) const expDocs = YAML.parseAllDocuments(outYaml, { mapAsMap: true }) From 5ff783d8d608aca627d083fa852945a44a9069fb Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 2 Jan 2021 16:34:49 +0200 Subject: [PATCH 53/89] Drop Document method listNonDefaultTags() BREAKING CHANGE: The public method is no longer available, as its internal use for it is being refactored to StreamDirectives. An alternative pattern will need to be documented for any current users, once the visitor API is available to use as a base for it. (#190) --- docs/04_documents.md | 1 - index.d.ts | 6 +----- src/doc/Document.d.ts | 6 +----- src/doc/Document.js | 10 +++------- 4 files changed, 5 insertions(+), 18 deletions(-) diff --git a/docs/04_documents.md b/docs/04_documents.md index b0ad88c5..82265df5 100644 --- a/docs/04_documents.md +++ b/docs/04_documents.md @@ -102,7 +102,6 @@ During stringification, a document with a true-ish `version` value will include | ------------------------------------------ | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | createNode(value, options?) | `Node` | Recursively wrap any input with appropriate `Node` containers. See [Creating Nodes](#creating-nodes) for more information. | | createPair(key, value, options?) | `Pair` | Recursively wrap `key` and `value` into a `Pair` object. See [Creating Nodes](#creating-nodes) for more information. | -| listNonDefaultTags() | `string[]` | List the tags used in the document that are not in the default `tag:yaml.org,2002:` namespace. | | parse(cst) | `Document` | Parse a CST into this document. Mostly an internal method, modifying the document according to the contents of the parsed `cst`. Calling this multiple times on a Document is not recommended. | | setSchema(id?, customTags?) | `void` | Set the schema used by the document. `id` may either be a YAML version, or the identifier of a YAML 1.2 schema; if set, `customTags` should have the same shape as the similarly-named option. | | setTagPrefix(handle, prefix) | `void` | Set `handle` as a shorthand string for the `prefix` tag namespace. | diff --git a/index.d.ts b/index.d.ts index 6b17369c..99f8cc3a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -283,11 +283,7 @@ export class Document extends Collection { * `Scalar` objects. */ createPair(key: any, value: any, options?: { wrapScalars?: boolean }): Pair - /** - * List the tags used in the document that are not in the default - * `tag:yaml.org,2002:` namespace. - */ - listNonDefaultTags(): string[] + /** Parse a CST into this document */ parse(cst: CST.Document): this /** diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts index 7bf391eb..48dcb1fe 100644 --- a/src/doc/Document.d.ts +++ b/src/doc/Document.d.ts @@ -87,11 +87,7 @@ export class Document extends Collection { * `Scalar` objects. */ createPair(key: any, value: any, options?: { wrapScalars?: boolean }): Pair - /** - * List the tags used in the document that are not in the default - * `tag:yaml.org,2002:` namespace. - */ - listNonDefaultTags(): string[] + /** Parse a CST into this document */ parse(cst: CST.Document): this /** diff --git a/src/doc/Document.js b/src/doc/Document.js index 52b088fa..433c5501 100644 --- a/src/doc/Document.js +++ b/src/doc/Document.js @@ -239,12 +239,6 @@ export class Document { return this } - listNonDefaultTags() { - return listTagNames(this.contents).filter( - t => t.indexOf(defaultTagPrefix) !== 0 - ) - } - setTagPrefix(handle, prefix) { if (handle[0] !== '!' || handle[handle.length - 1] !== '!') throw new Error('Handle must start and end with !') @@ -306,7 +300,9 @@ export class Document { lines.push(vd) hasDirectives = true } - const tagNames = this.listNonDefaultTags() + const tagNames = listTagNames(this.contents).filter( + t => t.indexOf(defaultTagPrefix) !== 0 + ) this.tagPrefixes.forEach(({ handle, prefix }) => { if (tagNames.some(t => t.indexOf(prefix) === 0)) { lines.push(`%TAG ${handle} ${prefix}`) From afe6ac6d0a80f5adef32a4d23981a4aa7bb5e5ce Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 2 Jan 2021 19:00:50 +0200 Subject: [PATCH 54/89] Use doc.directives.tags rather than doc.tagPrefixes Also add exports config for importing TS files into JS and refactor listTagNames as TS. --- rollup.dev-config.js | 2 + src/compose/compose-doc.ts | 2 +- src/compose/parse-docs.ts | 12 +++--- src/compose/resolve-props.ts | 2 +- src/doc/Document.d.ts | 5 +-- src/doc/Document.js | 23 +++-------- src/doc/{listTagNames.js => listTagNames.ts} | 6 +-- src/{compose => doc}/stream-directives.ts | 40 +++++++++++++++++--- src/stringify/stringify.js | 5 +-- src/stringify/stringifyTag.js | 28 -------------- tests/doc/YAML-1.2.spec.js | 13 ++----- tests/doc/types.js | 18 ++++----- tests/yaml-test-suite.js | 3 +- 13 files changed, 70 insertions(+), 89 deletions(-) rename src/doc/{listTagNames.js => listTagNames.ts} (65%) rename src/{compose => doc}/stream-directives.ts (70%) delete mode 100644 src/stringify/stringifyTag.js diff --git a/rollup.dev-config.js b/rollup.dev-config.js index fc0ac30c..4979abb1 100644 --- a/rollup.dev-config.js +++ b/rollup.dev-config.js @@ -1,3 +1,4 @@ +import { resolve } from 'path' import babel from '@rollup/plugin-babel' export default { @@ -7,6 +8,7 @@ export default { 'src/errors.js', 'src/options.js' ], + external: [resolve('src/doc/stream-directives.js')], output: { dir: 'lib', format: 'cjs', esModule: false, preserveModules: true }, plugins: [babel()] } diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index 529ee5bc..b6226ee7 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -1,10 +1,10 @@ import { Document } from '../doc/Document.js' +import { StreamDirectives } from '../doc/stream-directives.js' import type { Options } from '../options.js' import type * as Parser from '../parse/parser.js' import { composeNode } from './compose-node.js' import { resolveEnd } from './resolve-end.js' import { resolveProps } from './resolve-props.js' -import { StreamDirectives } from './stream-directives.js' export function composeDoc( options: Options | undefined, diff --git a/src/compose/parse-docs.ts b/src/compose/parse-docs.ts index 5970bcb9..b8263bfd 100644 --- a/src/compose/parse-docs.ts +++ b/src/compose/parse-docs.ts @@ -1,9 +1,9 @@ -import { Document } from '../doc/Document' -import { YAMLParseError, YAMLWarning } from '../errors' -import type { Options } from '../options' -import { Parser } from '../parse/parser' -import { composeDoc } from './compose-doc' -import { StreamDirectives } from './stream-directives' +import { Document } from '../doc/Document.js' +import { StreamDirectives } from '../doc/stream-directives.js' +import { YAMLParseError, YAMLWarning } from '../errors.js' +import type { Options } from '../options.js' +import { Parser } from '../parse/parser.js' +import { composeDoc } from './compose-doc.js' export function parseDocs(source: string, options?: Options) { const directives = new StreamDirectives() diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index 88878f3e..0bd2b8da 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -1,5 +1,5 @@ import type { Document } from '../doc/Document.js' -import { SourceToken } from '../parse/parser' +import { SourceToken } from '../parse/parser.js' function isSpaceBefore(sep: string) { if (!sep) return false diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts index 48dcb1fe..e2e8ff57 100644 --- a/src/doc/Document.d.ts +++ b/src/doc/Document.d.ts @@ -1,10 +1,10 @@ import { Alias, Collection, Merge, Node, Pair } from '../ast' -import { StreamDirectives } from '../compose/stream-directives' import { Type } from '../constants' import { CST } from '../cst' import { YAMLError, YAMLWarning } from '../errors' import { Options } from '../options' import { Schema } from './Schema' +import { StreamDirectives } from './stream-directives' type Replacer = any[] | ((key: any, value: any) => boolean) type Reviver = (key: any, value: any) => any @@ -38,7 +38,7 @@ export class Document extends Collection { constructor(value?: any, options?: Options) constructor(value: any, replacer: null | Replacer, options?: Options) - directives?: StreamDirectives + directives: StreamDirectives tag: never directivesEndMarker?: boolean @@ -130,7 +130,6 @@ export class Document extends Collection { export namespace Document { interface Parsed extends Document { contents: Node.Parsed | null - directives: StreamDirectives /** The schema used with the document. */ schema: Schema } diff --git a/src/doc/Document.js b/src/doc/Document.js index 433c5501..32e850af 100644 --- a/src/doc/Document.js +++ b/src/doc/Document.js @@ -9,7 +9,6 @@ import { toJS } from '../ast/index.js' import { Document as CSTDocument } from '../cst/Document.js' -import { defaultTagPrefix } from '../constants.js' import { YAMLError } from '../errors.js' import { defaultOptions, documentOptions } from '../options.js' import { addComment } from '../stringify/addComment.js' @@ -19,9 +18,9 @@ import { Anchors } from './Anchors.js' import { Schema } from './Schema.js' import { applyReviver } from './applyReviver.js' import { createNode } from './createNode.js' -import { listTagNames } from './listTagNames.js' import { parseContents } from './parseContents.js' import { parseDirectives } from './parseDirectives.js' +import { StreamDirectives } from './stream-directives.js' function assertCollection(contents) { if (contents instanceof Collection) return true @@ -46,6 +45,7 @@ export class Document { this.anchors = new Anchors(this.options.anchorPrefix) this.commentBefore = null this.comment = null + this.directives = new StreamDirectives() this.directivesEndMarker = null this.errors = [] this.schema = null @@ -291,24 +291,11 @@ export class Document { this.setSchema() const lines = [] let hasDirectives = false - if (this.version) { - let vd = '%YAML 1.2' - if (this.schema.name === 'yaml-1.1') { - if (this.version === '1.0') vd = '%YAML:1.0' - else if (this.version === '1.1') vd = '%YAML 1.1' - } - lines.push(vd) + const dir = this.directives.toString(this) + if (dir) { + lines.push(dir) hasDirectives = true } - const tagNames = listTagNames(this.contents).filter( - t => t.indexOf(defaultTagPrefix) !== 0 - ) - this.tagPrefixes.forEach(({ handle, prefix }) => { - if (tagNames.some(t => t.indexOf(prefix) === 0)) { - lines.push(`%TAG ${handle} ${prefix}`) - hasDirectives = true - } - }) if (hasDirectives || this.directivesEndMarker) lines.push('---') if (this.commentBefore) { if (hasDirectives || !this.directivesEndMarker) lines.unshift('') diff --git a/src/doc/listTagNames.js b/src/doc/listTagNames.ts similarity index 65% rename from src/doc/listTagNames.js rename to src/doc/listTagNames.ts index ec0a2432..d64d062b 100644 --- a/src/doc/listTagNames.js +++ b/src/doc/listTagNames.ts @@ -1,6 +1,6 @@ -import { Collection, Pair, Scalar } from '../ast/index.js' +import { Collection, Node, Pair, Scalar } from '../ast/index.js' -const visit = (node, tags) => { +function visit(node: Node, tags: Record) { if (node && typeof node === 'object') { const { tag } = node if (node instanceof Collection) { @@ -16,4 +16,4 @@ const visit = (node, tags) => { return tags } -export const listTagNames = node => Object.keys(visit(node, {})) +export const listTagNames = (node: Node) => Object.keys(visit(node, {})) diff --git a/src/compose/stream-directives.ts b/src/doc/stream-directives.ts similarity index 70% rename from src/compose/stream-directives.ts rename to src/doc/stream-directives.ts index 6ca298e0..b4f5f7bc 100644 --- a/src/compose/stream-directives.ts +++ b/src/doc/stream-directives.ts @@ -1,3 +1,18 @@ +import type { Document } from './Document.js' +import { listTagNames } from './listTagNames.js' + +const escapeChars: Record = { + '!': '%21', + ',': '%2C', + '[': '%5B', + ']': '%5D', + '{': '%7B', + '}': '%7D' +} + +const escapeTagName = (tn: string) => + tn.replace(/[!,[\]{}]/g, ch => escapeChars[ch]) + export class StreamDirectives { tags: Record = { '!!': 'tag:yaml.org,2002:' } yaml: { version: '1.1' | '1.2' | undefined } = { version: undefined } @@ -80,12 +95,27 @@ export class StreamDirectives { return null } - toString(includeVersion: boolean) { - let res = includeVersion ? `%YAML ${this.yaml.version}\n` : '' + /** + * Given a fully resolved tag, returns its printable string form, + * taking into account current tag prefixes and defaults. + */ + tagString(tag: string) { for (const [handle, prefix] of Object.entries(this.tags)) { - if (handle !== '!!' || prefix !== 'tag:yaml.org,2002:') - res += `%TAG ${handle} ${prefix}\n` + if (tag.startsWith(prefix)) + return handle + escapeTagName(tag.substring(prefix.length)) } - return res + return tag[0] === '!' ? tag : `!<${tag}>` + } + + toString(doc?: Document) { + const lines = + !doc || doc.version ? [`%YAML ${this.yaml.version || '1.2'}`] : [] + const tagNames = doc && listTagNames(doc.contents) + for (const [handle, prefix] of Object.entries(this.tags)) { + if (handle === '!!' && prefix === 'tag:yaml.org,2002:') continue + if (!tagNames || tagNames.some(tn => tn.startsWith(prefix))) + lines.push(`%TAG ${handle} ${prefix}`) + } + return lines.join('\n') } } diff --git a/src/stringify/stringify.js b/src/stringify/stringify.js index 8db1a52c..8e463e85 100644 --- a/src/stringify/stringify.js +++ b/src/stringify/stringify.js @@ -3,7 +3,6 @@ import { Node } from '../ast/Node.js' import { Pair } from '../ast/Pair.js' import { Scalar } from '../ast/Scalar.js' import { stringifyString } from './stringifyString.js' -import { stringifyTag } from './stringifyTag.js' function getTagObject(tags, item) { if (item instanceof Alias) return Alias @@ -40,9 +39,9 @@ function stringifyProps(node, tagObj, { anchors, doc }) { props.push(`&${anchor}`) } if (node.tag) { - props.push(stringifyTag(doc, node.tag)) + props.push(doc.directives.tagString(node.tag)) } else if (!tagObj.default) { - props.push(stringifyTag(doc, tagObj.tag)) + props.push(doc.directives.tagString(tagObj.tag)) } return props.join(' ') } diff --git a/src/stringify/stringifyTag.js b/src/stringify/stringifyTag.js deleted file mode 100644 index d325375b..00000000 --- a/src/stringify/stringifyTag.js +++ /dev/null @@ -1,28 +0,0 @@ -export function stringifyTag(doc, tag) { - if ((doc.version || doc.options.version) === '1.0') { - const priv = tag.match(/^tag:private\.yaml\.org,2002:([^:/]+)$/) - if (priv) return '!' + priv[1] - const vocab = tag.match(/^tag:([a-zA-Z0-9-]+)\.yaml\.org,2002:(.*)/) - return vocab ? `!${vocab[1]}/${vocab[2]}` : `!${tag.replace(/^tag:/, '')}` - } - - let p = doc.tagPrefixes.find(p => tag.indexOf(p.prefix) === 0) - if (!p) { - const dtp = doc.getDefaults().tagPrefixes - p = dtp && dtp.find(p => tag.indexOf(p.prefix) === 0) - } - if (!p) return tag[0] === '!' ? tag : `!<${tag}>` - const suffix = tag.substr(p.prefix.length).replace( - /[!,[\]{}]/g, - ch => - ({ - '!': '%21', - ',': '%2C', - '[': '%5B', - ']': '%5D', - '{': '%7B', - '}': '%7D' - }[ch]) - ) - return p.handle + suffix -} diff --git a/tests/doc/YAML-1.2.spec.js b/tests/doc/YAML-1.2.spec.js index 68fc5e43..123d6d17 100644 --- a/tests/doc/YAML-1.2.spec.js +++ b/tests/doc/YAML-1.2.spec.js @@ -994,9 +994,6 @@ Chomping: | --- foo`, tgt: ['foo'], - errors: [ - ['The %YAML directive must only be given at most once per document.'] - ], special: src => { const doc = YAML.parseDocument(src) expect(doc.version).toBe('1.1') @@ -1017,14 +1014,12 @@ foo`, --- bar`, tgt: ['bar'], - errors: [ - [ - 'The %TAG directive must only be given at most once per handle in the same document.' - ] - ], special: src => { const doc = YAML.parseDocument(src) - expect(doc.tagPrefixes).toMatchObject([{ handle: '!', prefix: '!foo' }]) + expect(doc.directives.tags).toMatchObject({ + '!!': 'tag:yaml.org,2002:', + '!': '!foo' + }) } }, diff --git a/tests/doc/types.js b/tests/doc/types.js index 0c8265e7..19168903 100644 --- a/tests/doc/types.js +++ b/tests/doc/types.js @@ -618,11 +618,13 @@ describe('custom tags', () => { test('modify', () => { const doc = YAML.parseDocument(src) const prefix = 'tag:example.com,2000:other/' - doc.setTagPrefix('!f!', prefix) - expect(doc.tagPrefixes).toMatchObject([ - { handle: '!e!' }, - { handle: '!f!' } - ]) + doc.directives.tags['!f!'] = prefix + expect(doc.directives.tags).toMatchObject({ + '!!': 'tag:yaml.org,2002:', + '!e!': 'tag:example.com,2000:test/', + '!f!': prefix + }) + doc.contents.commentBefore = 'c' doc.contents.items[3].comment = 'cc' const s = new Scalar(6) @@ -640,12 +642,6 @@ describe('custom tags', () => { - !f!w "4" - '5' #cc\n` ) - - doc.setTagPrefix('!f!', null) - expect(doc.tagPrefixes).toMatchObject([{ handle: '!e!' }]) - expect(() => doc.setTagPrefix('!f', prefix)).toThrow( - 'Handle must start and end with !' - ) }) describe('custom tag objects', () => { diff --git a/tests/yaml-test-suite.js b/tests/yaml-test-suite.js index 05754eea..9b8b085e 100644 --- a/tests/yaml-test-suite.js +++ b/tests/yaml-test-suite.js @@ -5,7 +5,8 @@ import * as YAML from '../index.js' import { testEvents } from '../dist/test-events.js' const skip = { - SF5V: ['errors'] // not erroring for duplicate %YAML directives + QLJ7: ['errors'], // allow %TAG directives to persist across documents + SF5V: ['errors'] // duplicate %YAML directives is not an error } const testDirs = fs From 05f5c75e083feabd5481be52db9bd6723638b09e Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 3 Jan 2021 04:28:45 +0200 Subject: [PATCH 55/89] Loads more debugging for various tests --- src/compose/resolve-block-map.ts | 9 ++- src/compose/resolve-block-scalar.ts | 2 +- src/compose/resolve-flow-collection.ts | 19 ++++-- src/compose/resolve-flow-scalar.ts | 9 +-- src/compose/validate-implicit-key.ts | 36 ++++++++++++ src/parse/lexer.ts | 81 ++++++++++++++++++++------ src/parse/parser.ts | 66 ++++++++++++--------- 7 files changed, 162 insertions(+), 60 deletions(-) create mode 100644 src/compose/validate-implicit-key.ts diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index fe7a488a..77621f99 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -5,6 +5,7 @@ import type { BlockMap } from '../parse/parser.js' import { composeNode } from './compose-node.js' import { resolveMergePair } from './resolve-merge-pair.js' import { resolveProps } from './resolve-props.js' +import { validateImplicitKey } from './validate-implicit-key.js' const startColMsg = 'All collection items must start at the same column' @@ -34,8 +35,12 @@ export function resolveBlockMap( if (key && 'indent' in key && key.indent !== indent) onError(offset, startColMsg) if (keyProps.anchor || keyProps.tagName || sep) { - // FIXME: check single-line - // FIXME: check 1024 chars + const err = validateImplicitKey(key) + if (err === 'single-line') + onError( + offset + keyProps.length, + 'Implicit keys need to be on a single line' + ) } else { // TODO: assert being at last item? if (keyProps.comment) { diff --git a/src/compose/resolve-block-scalar.ts b/src/compose/resolve-block-scalar.ts index b82d3fed..d428c8c6 100644 --- a/src/compose/resolve-block-scalar.ts +++ b/src/compose/resolve-block-scalar.ts @@ -25,7 +25,7 @@ export function resolveBlockScalar( // shortcut for empty contents if (!scalar.source || chompStart === 0) { const value = - header.chomp === '+' ? lines.map(line => line[0]).join('\n') + '\n' : '' + header.chomp === '+' ? lines.map(line => line[0]).join('\n') : '' let length = header.length if (scalar.source) length += scalar.source.length return { value, type, comment: header.comment, length } diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index 3d79c5ac..c98eb2a6 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -1,10 +1,11 @@ import { Node, Pair, YAMLMap, YAMLSeq } from '../ast/index.js' import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' -import type { FlowCollection, SourceToken } from '../parse/parser.js' +import type { FlowCollection, SourceToken, Token } from '../parse/parser.js' import { composeNode } from './compose-node.js' import { resolveEnd } from './resolve-end.js' import { resolveMergePair } from './resolve-merge-pair.js' +import { validateImplicitKey } from './validate-implicit-key.js' export function resolveFlowCollection( doc: Document.Parsed, @@ -32,6 +33,7 @@ export function resolveFlowCollection( let atExplicitKey = false let atValueEnd = false let nlAfterValueInSeq = false + let seqKeyToken: Token | null = null function getProps() { const props = { spaceBefore, comment, anchor, tagName } @@ -134,11 +136,16 @@ export function resolveFlowCollection( value = null } // else explicit key } else if (value) { - if (doc.options.strict && nlAfterValueInSeq) - onError( - offset, + if (doc.options.strict) { + const slMsg = 'Implicit keys of flow sequence pairs need to be on a single line' - ) + if (nlAfterValueInSeq) onError(offset, slMsg) + else if (seqKeyToken) { + const err = validateImplicitKey(seqKeyToken) + if (err === 'single-line') onError(offset, slMsg) + seqKeyToken = null + } + } key = value value = null } else { @@ -162,9 +169,11 @@ export function resolveFlowCollection( atValueEnd = true hasSpace = false nlAfterValueInSeq = false + seqKeyToken = null break default: { if (value) onError(offset, 'Missing , between flow collection items') + if (!isMap && !key && !atExplicitKey) seqKeyToken = token value = composeNode(doc, token, getProps(), onError) offset = value.range[1] isSourceToken = false diff --git a/src/compose/resolve-flow-scalar.ts b/src/compose/resolve-flow-scalar.ts index 7aa1871a..905ad5f1 100644 --- a/src/compose/resolve-flow-scalar.ts +++ b/src/compose/resolve-flow-scalar.ts @@ -78,10 +78,10 @@ function singleQuotedValue( } function foldLines(source: string) { - const lines = source.split(/[ \t]*\n[ \t]*/) - let res = '' - let sep = '' - for (let i = 0; i < lines.length; ++i) { + const lines = source.split(/[ \t]*\r?\n[ \t]*/) + let res = lines[0] + let sep = ' ' + for (let i = 1; i < lines.length - 1; ++i) { const line = lines[i] if (line === '') { if (sep === '\n') res += sep @@ -91,6 +91,7 @@ function foldLines(source: string) { sep = ' ' } } + if (lines.length > 1) res += sep + lines[lines.length - 1] return res } diff --git a/src/compose/validate-implicit-key.ts b/src/compose/validate-implicit-key.ts new file mode 100644 index 00000000..08caa0e9 --- /dev/null +++ b/src/compose/validate-implicit-key.ts @@ -0,0 +1,36 @@ +import type { Token } from '../parse/parser.js' + +function containsNewline(key: Token) { + switch (key.type) { + case 'alias': + case 'scalar': + case 'double-quoted-scalar': + case 'single-quoted-scalar': + return key.source.includes('\n') + case 'flow-collection': + for (const token of key.items) { + switch (token.type) { + case 'newline': + return true + case 'alias': + case 'scalar': + case 'double-quoted-scalar': + case 'single-quoted-scalar': + case 'flow-collection': + if (containsNewline(token)) return true + break + } + } + return false + default: + return true + } +} + +export function validateImplicitKey(key: Token | null | undefined) { + if (key) { + if (containsNewline(key)) return 'single-line' + // TODO: check 1024 chars + } + return null +} diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index fa029bc4..2b4c0e94 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -99,14 +99,44 @@ const isNotIdentifierChar = (ch: string) => export class Lexer { push: (token: string) => void + /** + * Flag indicating whether the end of the current buffer marks the end of + * all input + */ atEnd = false + + /** + * Explicit indent set in block scalar header, as an offset from the current + * minimum indent, so e.g. set to 1 from a header `|2+`. Set to -1 if not + * explicitly set. + */ + blockScalarIndent = -1 + + /** Current input */ buffer = '' - blockScalarIndent = -1 // non-negative if explicitly in header - flowKey = false // can : immediately follow this flow node - flowLevel = 0 // count of surrounding flow collection levels - indentNext = 0 // minimum indent level for next line + + /** + * Flag noting whether the map value indicator : can immediately follow this + * node within a flow context. + */ + flowKey = false + + /** Count of surrounding flow collection levels. */ + flowLevel = 0 + + /** + * Minimum level of indentation required for next lines to be parsed as a + * part of the current scalar value. + */ + indentNext = 0 + + /** Indentation level of the current line. */ indentValue = 0 // actual indent level of current line + + /** Stores the state of the lexer if reaching the end of incpomplete input */ next: State | null = null + + /** A pointer to `buffer`; the current position of the lexer. */ pos = 0 /** @@ -243,16 +273,16 @@ export class Lexer { const s = this.peek(3) if (s === '---' && isEmpty(this.charAt(3))) { this.pushCount(3) - this.indentNext = 0 this.indentValue = 0 + this.indentNext = 0 return 'doc' } else if (s === '...' && isEmpty(this.charAt(3))) { this.pushCount(3) return 'stream' } } - this.indentNext = this.pushSpaces(false) - this.indentValue = this.indentNext + this.indentValue = this.pushSpaces(false) + if (this.indentNext > this.indentValue) this.indentNext = this.indentValue return this.parseBlockStart() } @@ -261,11 +291,10 @@ export class Lexer { if (!ch1 && !this.atEnd) return this.setNext('block-start') if ((ch0 === '-' || ch0 === '?' || ch0 === ':') && isEmpty(ch1)) { const n = this.pushCount(1) + this.pushSpaces(true) - this.indentNext = this.indentValue + this.indentNext = this.indentValue + 1 this.indentValue += n return this.parseBlockStart() } - if (this.indentValue > this.indentNext) this.indentNext += 1 return 'doc' } @@ -275,9 +304,10 @@ export class Lexer { if (line === null) return this.setNext('doc') let n = this.pushIndicators() switch (line[n]) { - case undefined: case '#': this.pushCount(line.length - n) + // fallthrough + case undefined: this.pushNewline() return this.parseLineStart() case '{': @@ -310,15 +340,31 @@ export class Lexer { } parseFlowCollection() { - while (this.pushNewline() + this.pushSpaces(true) > 0) {} + let nl: number, sp: number + let indent = -1 + do { + nl = this.pushNewline() + sp = this.pushSpaces(true) + if (nl > 0) indent = sp + } while (nl + sp > 0) const line = this.getLine() if (line === null) return this.setNext('flow') + if ( + indent === 0 && + (line.startsWith('---') || line.startsWith('...')) && + isEmpty(line[3]) + ) { + // error: document marker within flow collection + this.flowLevel = 0 + return this.parseLineStart() + } let n = line[0] === ',' ? this.pushCount(1) + this.pushSpaces(true) : 0 n += this.pushIndicators() switch (line[n]) { - case undefined: case '#': - this.pushCount(line.length) + this.pushCount(line.length - n) + // fallthrough + case undefined: this.pushNewline() return 'flow' case '{': @@ -416,10 +462,7 @@ export class Lexer { break loop } } - if (ch && indent < this.indentNext) { - this.push(SCALAR) - this.push('') - } else { + if (indent >= this.indentNext) { if (this.blockScalarIndent === -1) this.indentNext = indent else this.indentNext += this.blockScalarIndent while (nl !== -1) { @@ -431,9 +474,9 @@ export class Lexer { if (!this.atEnd) return this.setNext('block-scalar') nl = this.buffer.length } - this.push(SCALAR) - this.pushToIndex(nl + 1, true) } + this.push(SCALAR) + this.pushToIndex(nl + 1, true) return this.parseLineStart() } diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 034dad7b..8c8b291e 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -105,6 +105,21 @@ function isFlowToken( } } +function getPrevProps(parent: Token) { + switch (parent.type) { + case 'document': + return parent.start + case 'block-map': { + const it = parent.items[parent.items.length - 1] + return it.sep || it.start + } + case 'block-seq': + return parent.items[parent.items.length - 1].start + default: + return [] + } +} + /** Note: May modify input array */ function getFirstKeyStartProps(prev: SourceToken[]) { if (prev.length === 0) return [] @@ -363,7 +378,7 @@ export class Parser { this.push(this.sourceToken) return } - const bv = this.startBlockValue() + const bv = this.startBlockValue(doc) if (bv) this.stack.push(bv) else { this.push({ @@ -377,23 +392,7 @@ export class Parser { scalar(scalar: FlowScalar) { if (this.type === 'map-value-ind') { - const parent = this.peek(2) - let prev: SourceToken[] - switch (parent.type) { - case 'document': - prev = parent.start - break - case 'block-map': { - const it = parent.items[parent.items.length - 1] - prev = it.sep || it.start - break - } - case 'block-seq': - prev = parent.items[parent.items.length - 1].start - break - default: - prev = [] - } + const prev = getPrevProps(this.peek(2)) const start = getFirstKeyStartProps(prev) let sep: SourceToken[] @@ -535,7 +534,7 @@ export class Parser { } default: { - const bv = this.startBlockValue() + const bv = this.startBlockValue(map) if (bv) return this.stack.push(bv) } } @@ -566,7 +565,7 @@ export class Parser { return } if (this.indent > seq.indent) { - const bv = this.startBlockValue() + const bv = this.startBlockValue(seq) if (bv) return this.stack.push(bv) } this.pop() @@ -599,7 +598,7 @@ export class Parser { fc.end.push(this.sourceToken) return } - const bv = this.startBlockValue() + const bv = this.startBlockValue(fc) if (bv) return this.stack.push(bv) this.pop() this.step() @@ -646,7 +645,7 @@ export class Parser { } as FlowScalar } - startBlockValue() { + startBlockValue(parent: Token) { switch (this.type) { case 'alias': case 'scalar': @@ -685,33 +684,42 @@ export class Parser { indent: this.indent, items: [{ start: [this.sourceToken] }] } as BlockMap - case 'map-value-ind': + case 'map-value-ind': { this.onKeyLine = true + const prev = getPrevProps(parent) + const start = getFirstKeyStartProps(prev) return { type: 'block-map', offset: this.offset, indent: this.indent, - items: [{ start: [], key: null, sep: [this.sourceToken] }] + items: [{ start, key: null, sep: [this.sourceToken] }] } as BlockMap + } } return null } lineEnd(token: Document | FlowCollection | FlowScalar) { switch (this.type) { + case 'comma': + case 'doc-start': + case 'doc-end': + case 'flow-seq-end': + case 'flow-map-end': + case 'map-value-ind': + this.pop() + this.step() + break case 'newline': this.onKeyLine = false // fallthrough case 'space': case 'comment': + default: + // all other values are errors if (token.end) token.end.push(this.sourceToken) else token.end = [this.sourceToken] if (this.type === 'newline') this.pop() - return - default: - this.pop() - this.step() - return } } } From dbe74f1c50a9d9258a362a51d7e9c3b1e9463d03 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 4 Jan 2021 01:08:57 +0200 Subject: [PATCH 56/89] Refactor doc-end parsing --- src/compose/parse-docs.ts | 27 +++++++++++++++++++++++++-- src/parse/parser.ts | 34 ++++++++++++++++++++++++++++------ tests/yaml-test-suite.js | 3 ++- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/src/compose/parse-docs.ts b/src/compose/parse-docs.ts index b8263bfd..27b32569 100644 --- a/src/compose/parse-docs.ts +++ b/src/compose/parse-docs.ts @@ -4,6 +4,7 @@ import { YAMLParseError, YAMLWarning } from '../errors.js' import type { Options } from '../options.js' import { Parser } from '../parse/parser.js' import { composeDoc } from './compose-doc.js' +import { resolveEnd } from './resolve-end.js' export function parseDocs(source: string, options?: Options) { const directives = new StreamDirectives() @@ -45,6 +46,8 @@ export function parseDocs(source: string, options?: Options) { atDirectives = false break } + case 'space': + break case 'comment': comment += token.source.substring(1) break @@ -60,9 +63,29 @@ export function parseDocs(source: string, options?: Options) { else docs[docs.length - 1].errors.push(error) break } - case 'space': - case 'doc-end': + case 'doc-end': { + const doc = docs[docs.length - 1] + if (!doc) { + const msg = 'Unexpected doc-end without preceding document' + errors.push(new YAMLParseError(token.offset, msg)) + break + } + const end = resolveEnd( + token.end, + token.offset + token.source.length, + doc.options.strict, + onError + ) + if (end.comment) { + if (doc.comment) doc.comment += `\n${end.comment}` + else doc.comment = end.comment + } + if (errors.length > 0) { + Array.prototype.push.apply(doc.errors, errors) + errors = [] + } break + } default: console.log('###', token) } diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 8c8b291e..4445058d 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -2,7 +2,7 @@ import { Lexer } from './lexer.js' import { SourceTokenType, prettyToken, tokenType } from './token-type.js' export interface SourceToken { - type: Exclude + type: Exclude indent: number source: string } @@ -27,6 +27,13 @@ export interface Document { end?: SourceToken[] } +export interface DocumentEnd { + type: 'doc-end' + offset: number + source: string + end?: SourceToken[] +} + export interface FlowScalar { type: 'alias' | 'scalar' | 'single-quoted-scalar' | 'double-quoted-scalar' offset: number @@ -79,6 +86,7 @@ export type Token = | ErrorToken | Directive | Document + | DocumentEnd | FlowScalar | BlockScalar | BlockMap @@ -251,6 +259,15 @@ export class Parser { step() { const top = this.peek(1) + if (this.type === 'doc-end' && top.type !== 'doc-end') { + while (this.stack.length > 0) this.pop() + this.stack.push({ + type: 'doc-end', + offset: this.offset, + source: this.source + }) + return + } if (!top) return this.stream() switch (top.type) { case 'document': @@ -268,6 +285,8 @@ export class Parser { return this.blockSequence(top) case 'flow-collection': return this.flowCollection(top) + case 'doc-end': + return this.documentEnd(top) } this.pop() // error } @@ -326,7 +345,6 @@ export class Parser { case 'directive-line': this.push({ type: 'directive', source: this.source }) return - case 'doc-end': case 'space': case 'comment': case 'newline': @@ -373,10 +391,6 @@ export class Parser { case 'newline': doc.start.push(this.sourceToken) return - case 'doc-end': - this.pop() - this.push(this.sourceToken) - return } const bv = this.startBlockValue(doc) if (bv) this.stack.push(bv) @@ -699,6 +713,14 @@ export class Parser { return null } + documentEnd(docEnd: DocumentEnd) { + if (this.type !== 'doc-mode') { + if (docEnd.end) docEnd.end.push(this.sourceToken) + else docEnd.end = [this.sourceToken] + if (this.type === 'newline') this.pop() + } + } + lineEnd(token: Document | FlowCollection | FlowScalar) { switch (this.type) { case 'comma': diff --git a/tests/yaml-test-suite.js b/tests/yaml-test-suite.js index 9b8b085e..2cb3adb8 100644 --- a/tests/yaml-test-suite.js +++ b/tests/yaml-test-suite.js @@ -5,8 +5,9 @@ import * as YAML from '../index.js' import { testEvents } from '../dist/test-events.js' const skip = { + B63P: ['errors'], // allow ... after directives QLJ7: ['errors'], // allow %TAG directives to persist across documents - SF5V: ['errors'] // duplicate %YAML directives is not an error + SF5V: ['errors'] // allow duplicate %YAML directives } const testDirs = fs From d4b3276cf4d263bdd418ab57f8c2ce03b90cb0a7 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 4 Jan 2021 01:09:47 +0200 Subject: [PATCH 57/89] Miscellaneous debugging --- src/compose/resolve-flow-collection.ts | 6 ++++-- src/compose/resolve-props.ts | 10 ++++++++++ src/doc/stream-directives.ts | 6 +++--- src/parse/lexer.ts | 8 ++++++-- src/parse/parser.ts | 12 +++++++++--- 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index c98eb2a6..0651a5ae 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -162,7 +162,9 @@ export function resolveFlowCollection( break } case 'comma': - addItem() + if (key || value || anchor || tagName || atExplicitKey) addItem() + else + onError(offset, `Unexpected , in flow ${isMap ? 'map' : 'sequence'}`) key = null value = null atExplicitKey = false @@ -183,7 +185,7 @@ export function resolveFlowCollection( } if (isSourceToken) offset += (token as SourceToken).source.length } - if (key || value || atExplicitKey) addItem() + if (key || value || anchor || tagName || atExplicitKey) addItem() const expectedEnd = isMap ? '}' : ']' const [ce, ...ee] = fc.end diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index 0bd2b8da..91040123 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -22,6 +22,7 @@ export function resolveProps( ) { let length = 0 let spaceBefore = false + let atNewline = startOnNewline let hasSpace = startOnNewline let comment = '' let hasComment = false @@ -33,6 +34,10 @@ export function resolveProps( for (const token of tokens) { switch (token.type) { case 'space': + // At the doc level, tabs at line start may be parsed as leading + // white space rather than indentation. + if (atNewline && indicator !== 'doc-start' && token.source[0] === '\t') + onError(offset + length, 'Tabs are not allowed as indentation') hasSpace = true break case 'comment': { @@ -51,6 +56,7 @@ export function resolveProps( break } case 'newline': + atNewline = true hasNewline = true hasSpace = true sep += token.source @@ -59,6 +65,7 @@ export function resolveProps( if (anchor) onError(offset + length, 'A node can have at most one anchor') anchor = token.source.substring(1) + atNewline = false hasSpace = false break case 'tag': { @@ -67,16 +74,19 @@ export function resolveProps( onError(offset, msg) ) if (tn) tagName = tn + atNewline = false hasSpace = false break } case indicator: // Could here handle preceding comments differently found = token.indent + atNewline = false hasSpace = false break default: onError(offset + length, `Unexpected ${token.type} token`) + atNewline = false hasSpace = false } if (token.source) length += token.source.length diff --git a/src/doc/stream-directives.ts b/src/doc/stream-directives.ts index b4f5f7bc..beb90b5a 100644 --- a/src/doc/stream-directives.ts +++ b/src/doc/stream-directives.ts @@ -28,7 +28,7 @@ export class StreamDirectives { * @param onError - May be called even if the action was successful * @returns `true` on success */ - add(line: string, onError: (offset: number, message: string) => void) { + add(line: string, onError: (offset: number, message: string, warning?: boolean) => void) { const parts = line.trim().split(/[ \t]+/) const name = parts.shift() switch (name) { @@ -51,12 +51,12 @@ export class StreamDirectives { this.yaml.version = version return true } else { - onError(6, `Unsupported YAML version ${version}`) + onError(6, `Unsupported YAML version ${version}`, true) return false } } default: - onError(0, `Unknown directive ${name}`) + onError(0, `Unknown directive ${name}`, true) return false } } diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 2b4c0e94..4261834e 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -358,7 +358,8 @@ export class Lexer { this.flowLevel = 0 return this.parseLineStart() } - let n = line[0] === ',' ? this.pushCount(1) + this.pushSpaces(true) : 0 + let n = 0 + while (line[n] === ',') n += this.pushCount(1) + this.pushSpaces(true) n += this.pushIndicators() switch (line[n]) { case '#': @@ -428,7 +429,10 @@ export class Lexer { end = nl - 1 } } - if (end === -1) return this.setNext('quoted-scalar') + if (end === -1) { + if (!this.atEnd) return this.setNext('quoted-scalar') + end = this.buffer.length + } this.pushToIndex(end + 1, false) return this.flowLevel ? 'flow' : 'doc' } diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 4445058d..6bb18505 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -234,10 +234,11 @@ export class Parser { if (this.onNewLine) this.onNewLine(this.offset + source.length) break case 'space': + if (this.atNewLine && source[0] === ' ') this.indent += source.length + break case 'explicit-key-ind': case 'map-value-ind': case 'seq-item-ind': - // TODO: also track parent indent if (this.atNewLine) this.indent += source.length break case 'doc-mode': @@ -618,20 +619,25 @@ export class Parser { this.step() } else { const parent = this.peek(2) - if (parent.type === 'block-map') { + if ( + (this.type === 'newline' || this.type == 'map-value-ind') && + parent.type === 'block-map' + ) { this.pop() this.step() } else if ( this.type === 'map-value-ind' && parent.type !== 'flow-collection' ) { + const prev = getPrevProps(parent) + const start = getFirstKeyStartProps(prev) const sep = fc.end.splice(1, fc.end.length) sep.push(this.sourceToken) const map: BlockMap = { type: 'block-map', offset: fc.offset, indent: fc.indent, - items: [{ start: [], key: fc, sep }] + items: [{ start, key: fc, sep }] } this.onKeyLine = true this.stack[this.stack.length - 1] = map From 14585f56a05429960d8e9bdbaac11f71a3f7e6c8 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Wed, 6 Jan 2021 15:25:34 +0200 Subject: [PATCH 58/89] Check for outer block indent when parsing flow collections Adds a new control character \x18 (Cancel) to lexer output, marking unexpected end of flow mode. --- src/parse/lexer.ts | 34 ++++++++++++++++++++++------------ src/parse/parser.ts | 8 +++++++- src/parse/token-type.ts | 5 +++++ 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 4261834e..0f5d7c23 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -66,7 +66,7 @@ plain-scalar(is-flow, min) [else] -> plain-scalar(min) */ -import { DOCUMENT, SCALAR } from './token-type.js' +import { DOCUMENT, FLOW_END, SCALAR } from './token-type.js' type State = | 'stream' @@ -345,28 +345,38 @@ export class Lexer { do { nl = this.pushNewline() sp = this.pushSpaces(true) - if (nl > 0) indent = sp + if (nl > 0) this.indentValue = indent = sp } while (nl + sp > 0) const line = this.getLine() if (line === null) return this.setNext('flow') if ( - indent === 0 && - (line.startsWith('---') || line.startsWith('...')) && - isEmpty(line[3]) + (indent !== -1 && indent < this.indentNext) || + (indent === 0 && + (line.startsWith('---') || line.startsWith('...')) && + isEmpty(line[3])) ) { - // error: document marker within flow collection - this.flowLevel = 0 - return this.parseLineStart() + // Allowing for the terminal ] or } at the same (rather than greater) + // indent level as the initial [ or { is technically invalid, but + // failing here would be surprising to users. + const atFlowEndMarker = + indent === this.indentNext - 1 && + this.flowLevel === 1 && + (line[0] === ']' || line[0] === '}') + if (!atFlowEndMarker) { + // this is an error + this.flowLevel = 0 + this.push(FLOW_END) + return this.parseLineStart() + } } let n = 0 while (line[n] === ',') n += this.pushCount(1) + this.pushSpaces(true) n += this.pushIndicators() switch (line[n]) { + case undefined: + return 'flow' case '#': this.pushCount(line.length - n) - // fallthrough - case undefined: - this.pushNewline() return 'flow' case '{': case '[': @@ -549,7 +559,7 @@ export class Lexer { case '?': // this is an error outside flow collections case '-': // this is an error if (isEmpty(this.charAt(1))) { - this.indentNext = this.indentValue + 1 + if (this.flowLevel === 0) this.indentNext = this.indentValue + 1 return ( this.pushCount(1) + this.pushSpaces(true) + this.pushIndicators() ) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 6bb18505..eeef6bc1 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -588,7 +588,13 @@ export class Parser { } flowCollection(fc: FlowCollection) { - if (fc.end.length === 0) { + if (this.type === 'flow-error-end') { + let top + do { + this.pop() + top = this.peek(1) + } while (top && top.type === 'flow-collection') + } else if (fc.end.length === 0) { switch (this.type) { case 'space': case 'comment': diff --git a/src/parse/token-type.ts b/src/parse/token-type.ts index 156ef8a0..bc0a37c8 100644 --- a/src/parse/token-type.ts +++ b/src/parse/token-type.ts @@ -1,4 +1,5 @@ export const DOCUMENT = '\x02' // Start of Text +export const FLOW_END = '\x18' // Cancel export const SCALAR = '\x1f' // Unit Separator export type SourceTokenType = @@ -20,6 +21,7 @@ export type SourceTokenType = | 'flow-map-end' | 'flow-seq-start' | 'flow-seq-end' + | 'flow-error-end' | 'comma' | 'single-quoted-scalar' | 'double-quoted-scalar' @@ -27,6 +29,7 @@ export type SourceTokenType = export function prettyToken(token: string) { if (token === DOCUMENT) return '' + if (token === FLOW_END) return '' if (token === SCALAR) return '' return JSON.stringify(token) } @@ -35,6 +38,8 @@ export function tokenType(source: string): SourceTokenType | null { switch (source) { case DOCUMENT: // start of doc-mode return 'doc-mode' + case FLOW_END: // unexpected end of flow mode + return 'flow-error-end' case SCALAR: // next token is a scalar value return 'scalar' case '---': From 8e722fff9334718e7811fb37f8f153fdd545c00c Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Wed, 6 Jan 2021 18:27:16 +0200 Subject: [PATCH 59/89] Fix tests to expect new error message strings --- tests/doc/YAML-1.1.spec.js | 4 +- tests/doc/YAML-1.2.spec.js | 113 +++++++++---------------------- tests/doc/errors.js | 135 +++++++++++++------------------------ tests/doc/types.js | 53 ++++++++------- 4 files changed, 108 insertions(+), 197 deletions(-) diff --git a/tests/doc/YAML-1.1.spec.js b/tests/doc/YAML-1.1.spec.js index ed61d3a4..4314ed1a 100644 --- a/tests/doc/YAML-1.1.spec.js +++ b/tests/doc/YAML-1.1.spec.js @@ -32,9 +32,7 @@ test('Use preceding directives if none defined', () => { const docs = YAML.parseAllDocuments(src, { prettyErrors: false }) expect(docs).toHaveLength(5) expect(docs.map(doc => doc.errors)).toMatchObject([[], [], [], [], []]) - const warn = tag => ({ - message: `The tag ${tag} is unavailable, falling back to tag:yaml.org,2002:str` - }) + const warn = tag => ({ message: `Unresolved tag: ${tag}` }) expect(docs.map(doc => doc.warnings)).toMatchObject([ [warn('!bar')], [warn('!foobar')], diff --git a/tests/doc/YAML-1.2.spec.js b/tests/doc/YAML-1.2.spec.js index 123d6d17..b6633b87 100644 --- a/tests/doc/YAML-1.2.spec.js +++ b/tests/doc/YAML-1.2.spec.js @@ -473,11 +473,7 @@ application specific tag: !something | 'The semantics of the tag\nabove may be different for\ndifferent documents.\n' } ], - warnings: [ - [ - 'The tag !something is unavailable, falling back to tag:yaml.org,2002:str' - ] - ], + warnings: [['Unresolved tag: !something']], special: src => { const doc = YAML.parseDocument(src, { schema: 'yaml-1.1' }) const data = doc.contents.items[1].value.value @@ -514,10 +510,10 @@ application specific tag: !something | ], warnings: [ [ - 'The tag tag:clarkevans.com,2002:circle is unavailable, falling back to tag:yaml.org,2002:map', - 'The tag tag:clarkevans.com,2002:line is unavailable, falling back to tag:yaml.org,2002:map', - 'The tag tag:clarkevans.com,2002:label is unavailable, falling back to tag:yaml.org,2002:map', - 'The tag tag:clarkevans.com,2002:shape is unavailable, falling back to tag:yaml.org,2002:seq' + 'Unresolved tag: tag:clarkevans.com,2002:circle', + 'Unresolved tag: tag:clarkevans.com,2002:line', + 'Unresolved tag: tag:clarkevans.com,2002:label', + 'Unresolved tag: tag:clarkevans.com,2002:shape' ] ] }, @@ -626,11 +622,7 @@ comments: 'Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338.' } ], - warnings: [ - [ - 'The tag tag:clarkevans.com,2002:invoice is unavailable, falling back to tag:yaml.org,2002:map' - ] - ] + warnings: [['Unresolved tag: tag:clarkevans.com,2002:invoice']] }, 'Example 2.28. Log File': { @@ -725,9 +717,7 @@ mapping: { sky: blue, sea: green }`, src: `anchored: !local &anchor value alias: *anchor`, tgt: [{ anchored: 'value', alias: 'value' }], - warnings: [ - ['The tag !local is unavailable, falling back to tag:yaml.org,2002:str'] - ], + warnings: [['Unresolved tag: !local']], special: src => { const tag = { tag: '!local', resolve: str => `local:${str}` } const res = YAML.parse(src, { customTags: [tag] }) @@ -971,7 +961,7 @@ Chomping: | # with a warning. --- "foo"`, tgt: ['foo'], - warnings: [['YAML only supports %TAG and %YAML directives, and not %FOO']] + warnings: [['Unknown directive %FOO']] } }, '6.8.1. “YAML” Directives': { @@ -981,7 +971,7 @@ Chomping: | --- "foo"`, tgt: ['foo'], - warnings: [['Document will be parsed as YAML 1.2 rather than YAML 1.3']], + warnings: [['Unsupported YAML version 1.3']], special: src => { const doc = YAML.parseDocument(src) expect(doc.version).toBe('1.3') @@ -1033,10 +1023,8 @@ bar`, !foo "bar"`, tgt: ['bar', 'bar'], warnings: [ - ['The tag !foo is unavailable, falling back to tag:yaml.org,2002:str'], - [ - 'The tag tag:example.com,2000:app/foo is unavailable, falling back to tag:yaml.org,2002:str' - ] + ['Unresolved tag: !foo'], + ['Unresolved tag: tag:example.com,2000:app/foo'] ], special: src => { const customTags = [ @@ -1059,11 +1047,7 @@ bar`, --- !!int 1 - 3 # Interval, not integer`, tgt: ['1 - 3'], - warnings: [ - [ - 'The tag tag:example.com,2000:app/int is unavailable, falling back to tag:yaml.org,2002:str' - ] - ], + warnings: [['Unresolved tag: tag:example.com,2000:app/int']], special: src => { const tag = { tag: 'tag:example.com,2000:app/int', @@ -1079,11 +1063,7 @@ bar`, --- !e!foo "bar"`, tgt: ['bar'], - warnings: [ - [ - 'The tag tag:example.com,2000:app/foo is unavailable, falling back to tag:yaml.org,2002:str' - ] - ], + warnings: [['Unresolved tag: tag:example.com,2000:app/foo']], special: src => { const tag = { tag: 'tag:example.com,2000:app/foo', @@ -1103,14 +1083,7 @@ bar`, --- # Color here !m!light green`, tgt: ['fluorescent', 'green'], - warnings: [ - [ - 'The tag !my-light is unavailable, falling back to tag:yaml.org,2002:str' - ], - [ - 'The tag !my-light is unavailable, falling back to tag:yaml.org,2002:str' - ] - ], + warnings: [['Unresolved tag: !my-light'], ['Unresolved tag: !my-light']], special: src => { const tag = { tag: '!my-light', resolve: str => `light:${str}` } const docs = YAML.parseAllDocuments(src, { customTags: [tag] }) @@ -1126,11 +1099,7 @@ bar`, --- - !e!foo "bar"`, tgt: [['bar']], - warnings: [ - [ - 'The tag tag:example.com,2000:app/foo is unavailable, falling back to tag:yaml.org,2002:str' - ] - ], + warnings: [['Unresolved tag: tag:example.com,2000:app/foo']], special: src => { const tag = { tag: 'tag:example.com,2000:app/foo', @@ -1153,9 +1122,7 @@ bar`, src: `! foo : ! baz`, tgt: [{ foo: 'baz' }], - warnings: [ - ['The tag !bar is unavailable, falling back to tag:yaml.org,2002:str'] - ], + warnings: [['Unresolved tag: !bar']], special: src => { const tag = { tag: '!bar', resolve: str => `bar${str}` } const res = YAML.parse(src, { customTags: [tag] }) @@ -1167,10 +1134,8 @@ bar`, src: `- ! foo - !<$:?> bar`, tgt: [['foo', 'bar']], - errors: [["Verbatim tags aren't resolved, so ! is invalid."]], - warnings: [ - ['The tag $:? is unavailable, falling back to tag:yaml.org,2002:str'] - ] + errors: [["Verbatim tags aren't resolved, so ! is invalid."]], + warnings: [['Unresolved tag: $:?']] }, 'Example 6.26. Tag Shorthands': { @@ -1182,8 +1147,8 @@ bar`, tgt: [['foo', 'bar', 'baz']], warnings: [ [ - 'The tag !local is unavailable, falling back to tag:yaml.org,2002:str', - 'The tag tag:example.com,2000:app/tag! is unavailable, falling back to tag:yaml.org,2002:str' + 'Unresolved tag: !local', + 'Unresolved tag: tag:example.com,2000:app/tag!' ] ], special: src => { @@ -1205,12 +1170,7 @@ bar`, - !e! foo - !h!bar baz`, tgt: [['foo', 'baz']], - errors: [ - [ - 'The !e! tag has no suffix.', - 'The !h! tag handle is non-default and was not declared.' - ] - ] + errors: [['Could not resolve tag: !h!bar', 'The !e! tag has no suffix.']] }, 'Example 6.28. Non-Specific Tags': { @@ -1555,15 +1515,13 @@ foo: bar --- - |2 ·text`.replace(/·/g, ' '), - tgt: [[' \ntext\n'], ['text text\n'], ['text\n']], + tgt: [[' \ntext\n'], ['text\n', 'text'], ['', 'text']], errors: [ [ 'Block scalars with more-indented leading empty lines must use an explicit indentation indicator' ], - ['Block scalars must not be less indented than their first line'], - [ - 'Block scalars must not be less indented than their explicit indentation indicator' - ] + ['Sequence item without - indicator'], + ['Sequence item without - indicator'] ] }, @@ -1755,9 +1713,7 @@ folded: >1 value`, tgt: [{ literal: 'value\n', folded: 'value\n' }], - warnings: [ - ['The tag !foo is unavailable, falling back to tag:yaml.org,2002:str'] - ], + warnings: [['Unresolved tag: !foo']], special: src => { const tag = { tag: '!foo', resolve: str => `foo${str}` } const res = YAML.parse(src, { customTags: [tag] }) @@ -1896,17 +1852,14 @@ for (const section in spec) { const json = documents.map(doc => doc.toJS()) expect(json).toMatchObject(tgt) documents.forEach((doc, i) => { - if (!errors || !errors[i]) expect(doc.errors).toHaveLength(0) - else - errors[i].forEach((err, j) => { - expect(doc.errors[j]).toBeInstanceOf(YAMLError) - expect(doc.errors[j].message).toBe(err) - }) - if (!warnings || !warnings[i]) expect(doc.warnings).toHaveLength(0) - else - warnings[i].forEach((err, j) => - expect(doc.warnings[j].message).toBe(err) - ) + expect(doc.errors.map(err => err.message)).toMatchObject( + (errors && errors[i]) || [] + ) + expect(doc.warnings.map(err => err.message)).toMatchObject( + (warnings && warnings[i]) || [] + ) + for (const err of doc.errors.concat(doc.warnings)) + expect(err).toBeInstanceOf(YAMLError) if (!jsWarnings) expect(mockWarn).not.toHaveBeenCalled() else { for (const warning of jsWarnings) diff --git a/tests/doc/errors.js b/tests/doc/errors.js index e86ffa06..590f839f 100644 --- a/tests/doc/errors.js +++ b/tests/doc/errors.js @@ -2,34 +2,19 @@ import { Node } from '../../src/cst/Node.js' import { YAMLError } from '../../src/errors.js' import * as YAML from '../../src/index.js' -let origPrettyErrors -beforeAll(() => { - origPrettyErrors = YAML.defaultOptions.prettyErrors - YAML.defaultOptions.prettyErrors = false -}) -afterAll(() => { - YAML.defaultOptions.prettyErrors = origPrettyErrors -}) - test('require a message and source for all errors', () => { const exp = /Invalid arguments/ expect(() => new YAMLError()).toThrow(exp) expect(() => new YAMLError('Foo')).toThrow(exp) expect(() => new YAMLError('Foo', {})).toThrow(exp) expect(() => new YAMLError('Foo', new Node())).toThrow(exp) - expect(() => new YAMLError('Foo', null, 'foo')).toThrow(exp) expect(() => new YAMLError('Foo', new Node(), 'foo')).not.toThrow() }) test('fail on map value indented with tab', () => { const src = 'a:\n\t1\nb:\n\t2\n' const doc = YAML.parseDocument(src) - expect(doc.errors).toMatchObject([ - { name: 'YAMLSemanticError' }, - { name: 'YAMLSemanticError' }, - { name: 'YAMLSemanticError' }, - { name: 'YAMLSemanticError' } - ]) + expect(doc.errors).not.toHaveLength(0) expect(() => String(doc)).toThrow( 'Document with errors cannot be stringified' ) @@ -38,23 +23,19 @@ test('fail on map value indented with tab', () => { test('eemeli/yaml#6', () => { const src = 'abc: 123\ndef' const doc = YAML.parseDocument(src) - expect(doc.errors).toMatchObject([{ name: 'YAMLSemanticError' }]) - const node = doc.errors[0].source - expect(node).toBeInstanceOf(Node) - expect(node.rangeAsLinePos).toMatchObject({ - start: { line: 2, col: 1 }, - end: { line: 2, col: 4 } - }) + expect(doc.errors).toMatchObject([{ name: 'YAMLParseError', offset: 9 }]) }) -describe('eemeli/yaml#7', () => { +describe.skip('eemeli/yaml#7', () => { test('map', () => { const src = '{ , }\n---\n{ 123,,, }\n' const docs = YAML.parseAllDocuments(src) - expect(docs[0].errors).toMatchObject([{ name: 'YAMLSyntaxError' }]) + expect(docs[0].errors).toMatchObject([ + { name: 'YAMLParseError', offset: 2 } + ]) expect(docs[1].errors).toMatchObject([ - { name: 'YAMLSyntaxError' }, - { name: 'YAMLSyntaxError' } + { name: 'YAMLParseError', offset: 16 }, + { name: 'YAMLParseError', offset: 17 } ]) const node = docs[0].errors[0].source expect(node).toBeInstanceOf(Node) @@ -66,10 +47,12 @@ describe('eemeli/yaml#7', () => { test('seq', () => { const src = '[ , ]\n---\n[ 123,,, ]\n' const docs = YAML.parseAllDocuments(src) - expect(docs[0].errors).toMatchObject([{ name: 'YAMLSyntaxError' }]) + expect(docs[0].errors).toMatchObject([ + { name: 'YAMLParseError', offset: 2 } + ]) expect(docs[1].errors).toMatchObject([ - { name: 'YAMLSyntaxError' }, - { name: 'YAMLSyntaxError' } + { name: 'YAMLParseError', offset: 16 }, + { name: 'YAMLParseError', offset: 17 } ]) const node = docs[1].errors[0].source expect(node).toBeInstanceOf(Node) @@ -113,7 +96,7 @@ describe('block collections', () => { const doc = YAML.parseDocument(src) expect(doc.errors).toMatchObject([ { message: 'A collection cannot be both a mapping and a sequence' }, - { message: 'Failed to resolve SEQ_ITEM node here' }, + { message: 'Implicit keys need to be on a single line' }, { message: 'Implicit map keys need to be followed by map values' } ]) expect(doc.contents).toMatchObject({ @@ -128,59 +111,49 @@ describe('block collections', () => { describe('missing flow collection terminator', () => { test('start only of flow map (eemeli/yaml#8)', () => { - const doc = YAML.parseDocument('{', { prettyErrors: true }) + const doc = YAML.parseDocument('{') expect(doc.errors).toMatchObject([ { - name: 'YAMLSemanticError', - message: - 'Expected flow map to end with } at line 1, column 2:\n\n{\n ^\n', - nodeType: 'FLOW_MAP', - range: { start: 1, end: 2 }, - linePos: { start: { line: 1, col: 2 }, end: { line: 1, col: 3 } } + name: 'YAMLParseError', + message: 'Expected flow map to end with }', + offset: 1 } ]) }) test('start only of flow sequence (eemeli/yaml#8)', () => { - const doc = YAML.parseDocument('[', { prettyErrors: true }) + const doc = YAML.parseDocument('[') expect(doc.errors).toMatchObject([ { - name: 'YAMLSemanticError', - message: - 'Expected flow sequence to end with ] at line 1, column 2:\n\n[\n ^\n', - nodeType: 'FLOW_SEQ', - range: { start: 1, end: 2 }, - linePos: { start: { line: 1, col: 2 }, end: { line: 1, col: 3 } } + name: 'YAMLParseError', + message: 'Expected flow sequence to end with ]', + offset: 1 } ]) }) test('flow sequence without end', () => { - const doc = YAML.parseDocument('[ foo, bar,', { prettyErrors: true }) + const doc = YAML.parseDocument('[ foo, bar,') expect(doc.errors).toMatchObject([ { - name: 'YAMLSemanticError', - message: - 'Expected flow sequence to end with ] at line 1, column 12:\n\n[ foo, bar,\n ^\n', - nodeType: 'FLOW_SEQ', - range: { start: 11, end: 12 }, - linePos: { start: { line: 1, col: 12 }, end: { line: 1, col: 13 } } + name: 'YAMLParseError', + message: 'Expected flow sequence to end with ]', + offset: 11 } ]) }) }) -describe('pretty errors', () => { +describe.skip('pretty errors', () => { test('eemeli/yaml#6', () => { const src = 'abc: 123\ndef' const doc = YAML.parseDocument(src, { prettyErrors: true }) expect(doc.errors).toMatchObject([ { - name: 'YAMLSemanticError', + name: 'YAMLParseError', message: 'Implicit map keys need to be followed by map values at line 2, column 1:\n\ndef\n^^^\n', - nodeType: 'PLAIN', - range: { start: 9, end: 12 }, + offset: 9, linePos: { start: { line: 2, col: 1 }, end: { line: 2, col: 4 } } } ]) @@ -192,30 +165,24 @@ describe('pretty errors', () => { const docs = YAML.parseAllDocuments(src, { prettyErrors: true }) expect(docs[0].errors).toMatchObject([ { - name: 'YAMLSyntaxError', - message: - 'Flow map contains an unexpected , at line 1, column 3:\n\n{ , }\n ^\n', - nodeType: 'FLOW_MAP', - range: { start: 2, end: 3 }, + name: 'YAMLParseError', + message: 'Unexpected , in flow map', + offset: 2, linePos: { start: { line: 1, col: 3 }, end: { line: 1, col: 4 } } } ]) expect(docs[0].errors[0]).not.toHaveProperty('source') expect(docs[1].errors).toMatchObject([ { - name: 'YAMLSyntaxError', - message: - 'Flow map contains an unexpected , at line 3, column 7:\n\n{ 123,,, }\n ^\n', - nodeType: 'FLOW_MAP', - range: { start: 16, end: 17 }, + name: 'YAMLParseError', + message: 'Unexpected , in flow map', + offset: 16, linePos: { start: { line: 3, col: 7 }, end: { line: 3, col: 8 } } }, { - name: 'YAMLSyntaxError', - message: - 'Flow map contains an unexpected , at line 3, column 8:\n\n{ 123,,, }\n ^\n', - nodeType: 'FLOW_MAP', - range: { start: 17, end: 18 }, + name: 'YAMLParseError', + message: 'Unexpected , in flow map', + offset: 17, linePos: { start: { line: 3, col: 8 }, end: { line: 3, col: 9 } } } ]) @@ -226,9 +193,7 @@ describe('pretty errors', () => { test('pretty warnings', () => { const src = '%FOO\n---bar\n' const doc = YAML.parseDocument(src, { prettyErrors: true }) - expect(doc.warnings).toMatchObject([ - { name: 'YAMLWarning', nodeType: 'DIRECTIVE' } - ]) + expect(doc.warnings).toMatchObject([{ name: 'YAMLWarning' }]) }) }) @@ -246,14 +211,9 @@ describe('invalid options', () => { test('broken document with comment before first node', () => { const doc = YAML.parseDocument('#c\n*x\nfoo\n') - expect(doc.contents).toBeNull() expect(doc.errors).toMatchObject([ - { name: 'YAMLReferenceError', message: 'Aliased anchor not found: x' }, - { - name: 'YAMLSyntaxError', - message: - 'Document contains trailing content not separated by a ... or --- line' - } + { name: 'YAMLParseError', message: 'Aliased anchor not found: x' }, + { name: 'YAMLParseError', message: 'Unexpected scalar at node end' } ]) }) @@ -261,23 +221,19 @@ describe('broken directives', () => { for (const tag of ['%TAG', '%YAML']) test(`incomplete ${tag} directive`, () => { const doc = YAML.parseDocument(`${tag}\n---\n`) - expect(doc.errors).toMatchObject([ - { name: 'YAMLSemanticError', source: { type: 'DIRECTIVE' } } - ]) + expect(doc.errors).toMatchObject([{ name: 'YAMLParseError', offset: 0 }]) }) test('missing separator', () => { const doc = YAML.parseDocument(`%YAML 1.2\n`) - expect(doc.errors).toMatchObject([ - { name: 'YAMLSemanticError', source: { type: 'DOCUMENT' } } - ]) + expect(doc.errors).toMatchObject([{ name: 'YAMLParseError', offset: 9 }]) }) }) test('multiple tags on one node', () => { const doc = YAML.parseDocument('!foo !bar baz\n') expect(doc.contents).toMatchObject({ value: 'baz', type: 'PLAIN' }) - expect(doc.errors).toMatchObject([{ name: 'YAMLSemanticError' }]) + expect(doc.errors).toMatchObject([{ name: 'YAMLParseError' }]) expect(doc.warnings).toMatchObject([{}]) }) @@ -291,8 +247,7 @@ describe('logLevel', () => { test('by default, warn for tag fallback', () => { YAML.parse('!foo bar') - const message = - 'The tag !foo is unavailable, falling back to tag:yaml.org,2002:str' + const message = 'Unresolved tag: !foo' expect(mock.mock.calls).toMatchObject([[{ message }]]) }) diff --git a/tests/doc/types.js b/tests/doc/types.js index 19168903..e744fa2d 100644 --- a/tests/doc/types.js +++ b/tests/doc/types.js @@ -32,13 +32,14 @@ describe('json schema', () => { expect(doc.toJS()).toMatchObject({ canonical: true, answer: false, - logical: null, - option: null + logical: 'True', + option: 'TruE' }) expect(doc.errors).toHaveLength(2) doc.errors = [] - doc.contents.items.splice(2, 2) - expect(String(doc)).toBe('"canonical": true\n"answer": false\n') + expect(String(doc)).toBe( + '"canonical": true\n"answer": false\n"logical": "True"\n"option": "TruE"\n' + ) }) test('!!float', () => { @@ -51,15 +52,14 @@ describe('json schema', () => { expect(doc.toJS()).toMatchObject({ canonical: 685230.15, fixed: 685230.15, - 'negative infinity': null, - 'not a number': null + 'negative infinity': '-.inf', + 'not a number': '.NaN' }) expect(doc.errors).toHaveLength(2) doc.errors = [] - doc.contents.items.splice(2, 2) doc.contents.items[1].value.tag = 'tag:yaml.org,2002:float' expect(String(doc)).toBe( - '"canonical": 685230.15\n"fixed": !!float 685230.15\n' + '"canonical": 685230.15\n"fixed": !!float 685230.15\n"negative infinity": "-.inf"\n"not a number": ".NaN"\n' ) }) @@ -73,15 +73,14 @@ describe('json schema', () => { expect(doc.toJS()).toMatchObject({ canonical: 685230, decimal: -685230, - octal: null, - hexadecimal: null + octal: '0o2472256', + hexadecimal: '0x0A74AE' }) expect(doc.errors).toHaveLength(2) doc.errors = [] - doc.contents.items.splice(2, 2) doc.set('bigint', 42n) expect(String(doc)).toBe( - '"canonical": 685230\n"decimal": -685230\n"bigint": 42\n' + '"canonical": 685230\n"decimal": -685230\n"octal": "0o2472256"\n"hexadecimal": "0x0A74AE"\n"bigint": 42\n' ) }) @@ -93,18 +92,17 @@ describe('json schema', () => { const doc = YAML.parseDocument(src, { schema: 'json' }) expect(doc.toJS()).toMatchObject({ - empty: null, - canonical: null, + empty: '', + canonical: '~', english: null, - '': 'null key' + '~': 'null key' }) - expect(doc.errors).toHaveLength(2) + expect(doc.errors).toHaveLength(3) doc.errors = [] - expect(String(doc)).toBe(`"empty": null -"canonical": null + expect(String(doc)).toBe(`"empty": "" +"canonical": "~" "english": null -? null -: "null key"\n`) +"~": "null key"\n`) }) }) @@ -508,7 +506,7 @@ date (00:00:00Z): 2002-12-14\n`) const doc = YAML.parseDocument(src, { version: '1.1' }) expect(doc.errors).toMatchObject([ { - name: 'YAMLSemanticError', + name: 'YAMLParseError', message: 'Ordered maps must not include duplicate keys: b' } ]) @@ -559,7 +557,7 @@ date (00:00:00Z): 2002-12-14\n`) const doc = YAML.parseDocument(src, { version: '1.1' }) expect(doc.errors).toMatchObject([ { - name: 'YAMLSemanticError', + name: 'YAMLParseError', message: 'Set items must all have null values' } ]) @@ -698,9 +696,16 @@ describe('schema changes', () => { version: '1.1' }) expect(doc.options.version).toBe('1.1') + expect(doc.directives.yaml).toMatchObject({ + version: '1.1', + explicit: false + }) doc.setSchema('1.2') - expect(doc.version).toBeNull() - expect(doc.options.version).toBe('1.2') + expect(doc.directives.yaml).toMatchObject({ + version: '1.2', + explicit: false + }) + expect(doc.options.version).toBe('1.1') expect(doc.options.schema).toBeUndefined() expect(() => String(doc)).toThrow(/Tag not resolved for Date value/) }) From 2be7e80d2ced613c15c7d377a7a7c2ebe1ef8aaa Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 9 Jan 2021 09:21:36 +0200 Subject: [PATCH 60/89] Refactor directives to support version-specific behaviour --- index.d.ts | 3 +- rollup.dev-config.js | 2 +- src/compose/compose-doc.ts | 7 +-- src/compose/parse-docs.ts | 8 ++- src/doc/Document.d.ts | 5 +- src/doc/Document.js | 12 ++-- .../{stream-directives.ts => directives.ts} | 61 ++++++++++++++++--- src/index.ts | 2 +- src/options.d.ts | 2 +- tests/doc/YAML-1.1.spec.js | 41 ++++++++----- tests/doc/YAML-1.2.spec.js | 44 +++++++++---- tests/yaml-test-suite.js | 1 - 12 files changed, 131 insertions(+), 57 deletions(-) rename src/doc/{stream-directives.ts => directives.ts} (63%) diff --git a/index.d.ts b/index.d.ts index 99f8cc3a..00d3d7aa 100644 --- a/index.d.ts +++ b/index.d.ts @@ -96,7 +96,7 @@ export interface Options extends Schema.Options { * * Default: `"1.2"` */ - version?: '1.0' | '1.1' | '1.2' + version?: '1.1' | '1.2' } /** @@ -326,6 +326,7 @@ export class Document extends Collection { export namespace Document { interface Parsed extends Document { contents: Node | null + range: [number, number] /** The schema used with the document. */ schema: Schema } diff --git a/rollup.dev-config.js b/rollup.dev-config.js index 4979abb1..651229cf 100644 --- a/rollup.dev-config.js +++ b/rollup.dev-config.js @@ -8,7 +8,7 @@ export default { 'src/errors.js', 'src/options.js' ], - external: [resolve('src/doc/stream-directives.js')], + external: [resolve('src/doc/directives.js')], output: { dir: 'lib', format: 'cjs', esModule: false, preserveModules: true }, plugins: [babel()] } diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index b6226ee7..3615ff9d 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -1,5 +1,5 @@ +import { Directives } from '../doc/directives.js' import { Document } from '../doc/Document.js' -import { StreamDirectives } from '../doc/stream-directives.js' import type { Options } from '../options.js' import type * as Parser from '../parse/parser.js' import { composeNode } from './compose-node.js' @@ -8,13 +8,12 @@ import { resolveProps } from './resolve-props.js' export function composeDoc( options: Options | undefined, - directives: StreamDirectives, + directives: Directives, { offset, start, value, end }: Parser.Document, onError: (offset: number, message: string, warning?: boolean) => void ) { const doc = new Document(undefined, options) as Document.Parsed - doc.directives = StreamDirectives.from(directives) - doc.version = directives.yaml.version + doc.directives = directives.atDocument() doc.setSchema() // FIXME: always do this in the constructor const props = resolveProps(doc, start, true, 'doc-start', offset, onError) diff --git a/src/compose/parse-docs.ts b/src/compose/parse-docs.ts index 27b32569..b92e9a85 100644 --- a/src/compose/parse-docs.ts +++ b/src/compose/parse-docs.ts @@ -1,13 +1,15 @@ +import { Directives } from '../doc/directives.js' import { Document } from '../doc/Document.js' -import { StreamDirectives } from '../doc/stream-directives.js' import { YAMLParseError, YAMLWarning } from '../errors.js' -import type { Options } from '../options.js' +import { defaultOptions, Options } from '../options.js' import { Parser } from '../parse/parser.js' import { composeDoc } from './compose-doc.js' import { resolveEnd } from './resolve-end.js' export function parseDocs(source: string, options?: Options) { - const directives = new StreamDirectives() + const directives = new Directives({ + version: options?.version || defaultOptions.version || '1.2' + }) const docs: Document.Parsed[] = [] const lines: number[] = [] diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts index e2e8ff57..49b97878 100644 --- a/src/doc/Document.d.ts +++ b/src/doc/Document.d.ts @@ -3,8 +3,8 @@ import { Type } from '../constants' import { CST } from '../cst' import { YAMLError, YAMLWarning } from '../errors' import { Options } from '../options' +import { Directives } from './directives' import { Schema } from './Schema' -import { StreamDirectives } from './stream-directives' type Replacer = any[] | ((key: any, value: any) => boolean) type Reviver = (key: any, value: any) => any @@ -38,7 +38,7 @@ export class Document extends Collection { constructor(value?: any, options?: Options) constructor(value: any, replacer: null | Replacer, options?: Options) - directives: StreamDirectives + directives: Directives tag: never directivesEndMarker?: boolean @@ -130,6 +130,7 @@ export class Document extends Collection { export namespace Document { interface Parsed extends Document { contents: Node.Parsed | null + range: [number, number] /** The schema used with the document. */ schema: Schema } diff --git a/src/doc/Document.js b/src/doc/Document.js index 32e850af..4d5b04be 100644 --- a/src/doc/Document.js +++ b/src/doc/Document.js @@ -18,9 +18,9 @@ import { Anchors } from './Anchors.js' import { Schema } from './Schema.js' import { applyReviver } from './applyReviver.js' import { createNode } from './createNode.js' +import { Directives } from './directives.js' import { parseContents } from './parseContents.js' import { parseDirectives } from './parseDirectives.js' -import { StreamDirectives } from './stream-directives.js' function assertCollection(contents) { if (contents instanceof Collection) return true @@ -45,12 +45,11 @@ export class Document { this.anchors = new Anchors(this.options.anchorPrefix) this.commentBefore = null this.comment = null - this.directives = new StreamDirectives() + this.directives = new Directives({ version: this.options.version }) this.directivesEndMarker = null this.errors = [] this.schema = null this.tagPrefixes = [] - this.version = null this.warnings = [] if (value === undefined) { @@ -140,7 +139,7 @@ export class Document { getDefaults() { return ( - Document.defaults[this.version] || + Document.defaults[this.directives.yaml.version] || Document.defaults[this.options.version] || {} ) @@ -197,9 +196,8 @@ export class Document { setSchema(id, customTags) { if (!id && !customTags && this.schema) return if (typeof id === 'number') id = id.toFixed(1) - if (id === '1.0' || id === '1.1' || id === '1.2') { - if (this.version) this.version = id - else this.options.version = id + if (id === '1.1' || id === '1.2') { + this.directives.yaml.version = id delete this.options.schema } else if (id && typeof id === 'string') { this.options.schema = id diff --git a/src/doc/stream-directives.ts b/src/doc/directives.ts similarity index 63% rename from src/doc/stream-directives.ts rename to src/doc/directives.ts index beb90b5a..2f63c2fb 100644 --- a/src/doc/stream-directives.ts +++ b/src/doc/directives.ts @@ -13,14 +13,45 @@ const escapeChars: Record = { const escapeTagName = (tn: string) => tn.replace(/[!,[\]{}]/g, ch => escapeChars[ch]) -export class StreamDirectives { - tags: Record = { '!!': 'tag:yaml.org,2002:' } - yaml: { version: '1.1' | '1.2' | undefined } = { version: undefined } +export class Directives { + static defaultYaml: Directives['yaml'] = { explicit: false, version: '1.2' } + static defaultTags: Directives['tags'] = { '!!': 'tag:yaml.org,2002:' } - static from(src: StreamDirectives) { - const res = new StreamDirectives() - Object.assign(res.tags, src.tags) - Object.assign(res.yaml, src.yaml) + yaml: { version: '1.1' | '1.2'; explicit?: boolean } + tags: Record + + /** + * Used when parsing YAML 1.1, where: + * > If the document specifies no directives, it is parsed using the same + * > settings as the previous document. If the document does specify any + * > directives, all directives of previous documents, if any, are ignored. + */ + private atNextDocument = false + + constructor(yaml?: Directives['yaml'], tags?: Directives['tags']) { + this.yaml = Object.assign({}, Directives.defaultYaml, yaml) + this.tags = Object.assign({}, Directives.defaultTags, tags) + } + + /** + * During parsing, get a Directives instance for the current document and + * update the stream state according to the current version's spec. + */ + atDocument() { + const res = new Directives(this.yaml, this.tags) + switch (this.yaml.version) { + case '1.1': + this.atNextDocument = true + break + case '1.2': + this.atNextDocument = false + this.yaml = { + explicit: Directives.defaultYaml.explicit, + version: '1.2' + } + this.tags = Object.assign({}, Directives.defaultTags) + break + } return res } @@ -28,7 +59,15 @@ export class StreamDirectives { * @param onError - May be called even if the action was successful * @returns `true` on success */ - add(line: string, onError: (offset: number, message: string, warning?: boolean) => void) { + add( + line: string, + onError: (offset: number, message: string, warning?: boolean) => void + ) { + if (this.atNextDocument) { + this.yaml = { explicit: Directives.defaultYaml.explicit, version: '1.1' } + this.tags = Object.assign({}, Directives.defaultTags) + this.atNextDocument = false + } const parts = line.trim().split(/[ \t]+/) const name = parts.shift() switch (name) { @@ -42,6 +81,7 @@ export class StreamDirectives { return true } case '%YAML': { + this.yaml.explicit = true if (parts.length < 1) { onError(0, '%YAML directive should contain exactly one part') return false @@ -108,8 +148,9 @@ export class StreamDirectives { } toString(doc?: Document) { - const lines = - !doc || doc.version ? [`%YAML ${this.yaml.version || '1.2'}`] : [] + const lines = this.yaml.explicit + ? [`%YAML ${this.yaml.version || '1.2'}`] + : [] const tagNames = doc && listTagNames(doc.contents) for (const [handle, prefix] of Object.entries(this.tags)) { if (handle === '!!' && prefix === 'tag:yaml.org,2002:') continue diff --git a/src/index.ts b/src/index.ts index deb76513..6e8d71cb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,7 +20,7 @@ export function parseDocument(src: string, options?: Options) { ) { const errMsg = 'Source contains multiple documents; please use YAML.parseAllDocuments()' - doc.errors.unshift(new YAMLSemanticError(-1, errMsg)) + doc.errors.push(new YAMLSemanticError(docs[1].range[0], errMsg)) } return doc } diff --git a/src/options.d.ts b/src/options.d.ts index 5138e29a..07d22629 100644 --- a/src/options.d.ts +++ b/src/options.d.ts @@ -91,7 +91,7 @@ export interface Options extends Schema.Options { * * Default: `"1.2"` */ - version?: '1.0' | '1.1' | '1.2' + version?: '1.1' | '1.2' } /** diff --git a/tests/doc/YAML-1.1.spec.js b/tests/doc/YAML-1.1.spec.js index 4314ed1a..7fe51e38 100644 --- a/tests/doc/YAML-1.1.spec.js +++ b/tests/doc/YAML-1.1.spec.js @@ -30,22 +30,33 @@ test('Use preceding directives if none defined', () => { !bar "Using previous YAML directive" ` const docs = YAML.parseAllDocuments(src, { prettyErrors: false }) - expect(docs).toHaveLength(5) - expect(docs.map(doc => doc.errors)).toMatchObject([[], [], [], [], []]) const warn = tag => ({ message: `Unresolved tag: ${tag}` }) - expect(docs.map(doc => doc.warnings)).toMatchObject([ - [warn('!bar')], - [warn('!foobar')], - [warn('!foobar')], - [warn('!bar')], - [warn('!bar')] - ]) - expect(docs.map(doc => doc.version)).toMatchObject([ - null, - null, - null, - '1.1', - '1.1' + expect(docs).toMatchObject([ + { + directives: { yaml: { version: '1.1', explicit: false } }, + errors: [], + warnings: [warn('!bar')] + }, + { + directives: { yaml: { version: '1.1', explicit: false } }, + errors: [], + warnings: [warn('!foobar')] + }, + { + directives: { yaml: { version: '1.1', explicit: false } }, + errors: [], + warnings: [warn('!foobar')] + }, + { + directives: { yaml: { version: '1.1', explicit: true } }, + errors: [], + warnings: [warn('!bar')] + }, + { + directives: { yaml: { version: '1.1', explicit: true } }, + errors: [], + warnings: [warn('!bar')] + } ]) expect(docs.map(String)).toMatchObject([ '!bar "First document"\n', diff --git a/tests/doc/YAML-1.2.spec.js b/tests/doc/YAML-1.2.spec.js index b6633b87..81726e75 100644 --- a/tests/doc/YAML-1.2.spec.js +++ b/tests/doc/YAML-1.2.spec.js @@ -756,7 +756,10 @@ double: "text"`, tgt: ['text'], special: src => { const doc = YAML.parseDocument(src) - expect(doc.version).toBe('1.2') + expect(doc.directives.yaml).toMatchObject({ + version: '1.2', + explicit: true + }) } }, @@ -974,7 +977,10 @@ Chomping: | warnings: [['Unsupported YAML version 1.3']], special: src => { const doc = YAML.parseDocument(src) - expect(doc.version).toBe('1.3') + expect(doc.directives.yaml).toMatchObject({ + version: '1.2', + explicit: true + }) } }, @@ -986,7 +992,10 @@ foo`, tgt: ['foo'], special: src => { const doc = YAML.parseDocument(src) - expect(doc.version).toBe('1.1') + expect(doc.directives.yaml).toMatchObject({ + version: '1.1', + explicit: true + }) } } }, @@ -1760,7 +1769,11 @@ Document`, Document ... # Suffix`, tgt: ['Document'], - special: src => expect(YAML.parseDocument(src).version).toBe('1.2') + special: src => + expect(YAML.parseDocument(src).directives.yaml).toMatchObject({ + version: '1.2', + explicit: true + }) }, 'Example 9.3. Bare Documents': { @@ -1795,10 +1808,14 @@ document # Empty ...`, tgt: ['%!PS-Adobe-2.0\n', null], - special: src => - YAML.parseAllDocuments(src).forEach(doc => - expect(doc.version).toBe('1.2') - ) + special(src) { + expect( + YAML.parseAllDocuments(src).map(doc => doc.directives.yaml) + ).toMatchObject([ + { version: '1.2', explicit: true }, + { version: '1.2', explicit: true } + ]) + } } }, @@ -1812,9 +1829,14 @@ document --- matches %: 20`, tgt: ['Document', null, { 'matches %': 20 }], - special: src => { - const versions = YAML.parseAllDocuments(src).map(doc => doc.version) - expect(versions).toMatchObject([null, null, '1.2']) + special(src) { + expect( + YAML.parseAllDocuments(src).map(doc => doc.directives.yaml) + ).toMatchObject([ + { version: '1.2', explicit: false }, + { version: '1.2', explicit: false }, + { version: '1.2', explicit: true } + ]) } } } diff --git a/tests/yaml-test-suite.js b/tests/yaml-test-suite.js index 2cb3adb8..580bdbd4 100644 --- a/tests/yaml-test-suite.js +++ b/tests/yaml-test-suite.js @@ -6,7 +6,6 @@ import { testEvents } from '../dist/test-events.js' const skip = { B63P: ['errors'], // allow ... after directives - QLJ7: ['errors'], // allow %TAG directives to persist across documents SF5V: ['errors'] // allow duplicate %YAML directives } From 816db58f31018a8fddefff878a12a299340062b6 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 9 Jan 2021 12:16:34 +0200 Subject: [PATCH 61/89] Allow for byte-order-mark in stream --- src/compose/parse-docs.ts | 1 + src/parse/lexer.ts | 8 ++++++-- src/parse/parser.ts | 1 + src/parse/token-type.ts | 24 ++++++++++++++++++------ 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/compose/parse-docs.ts b/src/compose/parse-docs.ts index b92e9a85..1853bffe 100644 --- a/src/compose/parse-docs.ts +++ b/src/compose/parse-docs.ts @@ -48,6 +48,7 @@ export function parseDocs(source: string, options?: Options) { atDirectives = false break } + case 'byte-order-mark': case 'space': break case 'comment': diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 0f5d7c23..544783a5 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -66,7 +66,7 @@ plain-scalar(is-flow, min) [else] -> plain-scalar(min) */ -import { DOCUMENT, FLOW_END, SCALAR } from './token-type.js' +import { BOM, DOCUMENT, FLOW_END, SCALAR } from './token-type.js' type State = | 'stream' @@ -237,8 +237,12 @@ export class Lexer { } parseStream() { - const line = this.getLine() + let line = this.getLine() if (line === null) return this.setNext('stream') + if (line[0] === BOM) { + this.pushCount(1) + line = line.substring(1) + } if (line[0] === '%') { let dirEnd = line.length const cs = line.indexOf('#') diff --git a/src/parse/parser.ts b/src/parse/parser.ts index eeef6bc1..769f36e2 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -346,6 +346,7 @@ export class Parser { case 'directive-line': this.push({ type: 'directive', source: this.source }) return + case 'byte-order-mark': case 'space': case 'comment': case 'newline': diff --git a/src/parse/token-type.ts b/src/parse/token-type.ts index bc0a37c8..1f5d3c97 100644 --- a/src/parse/token-type.ts +++ b/src/parse/token-type.ts @@ -1,8 +1,17 @@ -export const DOCUMENT = '\x02' // Start of Text -export const FLOW_END = '\x18' // Cancel -export const SCALAR = '\x1f' // Unit Separator +/** The byte order mark */ +export const BOM = '\u{FEFF}' + +/** Start of doc-mode */ +export const DOCUMENT = '\x02' // C0: Start of Text + +/** Unexpected end of flow-mode */ +export const FLOW_END = '\x18' // C0: Cancel + +/** Next token is a scalar value */ +export const SCALAR = '\x1f' // C0: Unit Separator export type SourceTokenType = + | 'byte-order-mark' | 'doc-mode' | 'scalar' | 'doc-start' @@ -28,6 +37,7 @@ export type SourceTokenType = | 'block-scalar-header' export function prettyToken(token: string) { + if (token === BOM) return '' if (token === DOCUMENT) return '' if (token === FLOW_END) return '' if (token === SCALAR) return '' @@ -36,11 +46,13 @@ export function prettyToken(token: string) { export function tokenType(source: string): SourceTokenType | null { switch (source) { - case DOCUMENT: // start of doc-mode + case BOM: + return 'byte-order-mark' + case DOCUMENT: return 'doc-mode' - case FLOW_END: // unexpected end of flow mode + case FLOW_END: return 'flow-error-end' - case SCALAR: // next token is a scalar value + case SCALAR: return 'scalar' case '---': return 'doc-start' From fb1a65a04f93fe85949d309abeec7bb18976dbb6 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 9 Jan 2021 12:18:06 +0200 Subject: [PATCH 62/89] Refactor parseDocs() into composeStream(), adding empty-stream handling --- .../{parse-docs.ts => compose-stream.ts} | 55 ++++++++++++++++++- src/index.ts | 8 ++- tests/doc/YAML-1.2.spec.js | 12 +++- tests/doc/errors.js | 2 +- 4 files changed, 68 insertions(+), 9 deletions(-) rename src/compose/{parse-docs.ts => compose-stream.ts} (63%) diff --git a/src/compose/parse-docs.ts b/src/compose/compose-stream.ts similarity index 63% rename from src/compose/parse-docs.ts rename to src/compose/compose-stream.ts index 1853bffe..9e833629 100644 --- a/src/compose/parse-docs.ts +++ b/src/compose/compose-stream.ts @@ -6,7 +6,23 @@ import { Parser } from '../parse/parser.js' import { composeDoc } from './compose-doc.js' import { resolveEnd } from './resolve-end.js' -export function parseDocs(source: string, options?: Options) { +export interface EmptyStream extends Array { + empty: true + comment: string + errors: YAMLParseError[] + warnings: YAMLWarning[] +} + +/** + * @returns If an empty `docs` array is returned, it will be of type + * EmptyStream. In TypeScript, you may use `'empty' in docs` as a + * type guard for it, as `docs.length === 0` won't catch it. + */ +export function composeStream( + source: string, + forceDoc: boolean, + options?: Options +) { const directives = new Directives({ version: options?.version || defaultOptions.version || '1.2' }) @@ -90,12 +106,45 @@ export function parseDocs(source: string, options?: Options) { break } default: - console.log('###', token) + errors.push(new YAMLParseError(-1, `Unsupported token ${token.type}`)) } }, n => lines.push(n) ) parser.parse(source) - return docs + if (docs.length === 0) { + if (forceDoc) { + const doc = new Document(undefined, options) as Document.Parsed + doc.directives = directives.atDocument() + if (atDirectives) { + const errMsg = 'Missing directives-end indicator line' + doc.errors.push(new YAMLParseError(source.length, errMsg)) + } + doc.setSchema() // FIXME: always do this in the constructor + doc.range = [0, source.length] + docs.push(doc) + } else { + const empty: EmptyStream = Object.assign( + [], + { empty: true } as { empty: true }, + { comment: comment.trimRight(), errors, warnings } + ) + return empty + } + } + + if (comment || errors.length > 0 || warnings.length > 0) { + const lastDoc = docs[docs.length - 1] + comment = comment.trimRight() + if (comment) { + if (lastDoc.comment) lastDoc.comment += `\n${comment}` + else lastDoc.comment = comment + } + Array.prototype.push.apply(lastDoc.errors, errors) + Array.prototype.push.apply(lastDoc.warnings, warnings) + } + + // TS would complain without the cast, but docs is always non-empty here + return (docs as unknown) as [Document.Parsed, ...Document.Parsed[]] } diff --git a/src/index.ts b/src/index.ts index 6e8d71cb..4f5873cc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -import { parseDocs } from './compose/parse-docs.js' +import { composeStream } from './compose/compose-stream.js' import { LogLevel } from './constants.js' import { Document } from './doc/Document.js' import { YAMLSemanticError } from './errors.js' @@ -8,10 +8,12 @@ import { Options } from './options.js' export { defaultOptions, scalarOptions } from './options.js' export { Document } -export const parseAllDocuments = parseDocs +export function parseAllDocuments(src: string, options?: Options) { + return composeStream(src, false, options) +} export function parseDocument(src: string, options?: Options) { - const docs = parseDocs(src, options) + const docs = composeStream(src, true, options) if (docs.length === 0) return null const doc = docs[0] if ( diff --git a/tests/doc/YAML-1.2.spec.js b/tests/doc/YAML-1.2.spec.js index 81726e75..faccc4ed 100644 --- a/tests/doc/YAML-1.2.spec.js +++ b/tests/doc/YAML-1.2.spec.js @@ -710,7 +710,11 @@ mapping: { sky: blue, sea: green }`, 'Example 5.5. Comment Indicator': { src: `# Comment only.`, - tgt: [null] + tgt: [], + special(src) { + const doc = YAML.parseDocument(src) + expect(doc.comment).toBe(' Comment only.') + } }, 'Example 5.6. Node Property Indicators': { @@ -936,7 +940,11 @@ Chomping: | 'Example 6.10. Comment Lines': { src: ` # Comment \n\n`, - tgt: [null] + tgt: [], + special(src) { + const doc = YAML.parseDocument(src) + expect(doc.comment).toBe(' Comment') + } }, 'Example 6.11. Multi-Line Comments': { diff --git a/tests/doc/errors.js b/tests/doc/errors.js index 590f839f..c2dab4ce 100644 --- a/tests/doc/errors.js +++ b/tests/doc/errors.js @@ -226,7 +226,7 @@ describe('broken directives', () => { test('missing separator', () => { const doc = YAML.parseDocument(`%YAML 1.2\n`) - expect(doc.errors).toMatchObject([{ name: 'YAMLParseError', offset: 9 }]) + expect(doc.errors).toMatchObject([{ name: 'YAMLParseError', offset: 10 }]) }) }) From f16333d630da77df20e2b461941c1f12c69fe5b0 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 9 Jan 2021 13:33:47 +0200 Subject: [PATCH 63/89] Fix bugs, mostly with tag parsing and error reporting --- src/compose/resolve-block-map.ts | 13 ++++++++++--- src/compose/resolve-block-seq.ts | 8 ++++++-- src/compose/resolve-flow-collection.ts | 2 +- src/doc/directives.ts | 3 ++- src/parse/lexer.ts | 3 ++- src/parse/parser.ts | 8 +++++--- tests/doc/YAML-1.2.spec.js | 3 ++- tests/doc/errors.js | 10 +++++----- 8 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index 77621f99..338ecec3 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -7,7 +7,7 @@ import { resolveMergePair } from './resolve-merge-pair.js' import { resolveProps } from './resolve-props.js' import { validateImplicitKey } from './validate-implicit-key.js' -const startColMsg = 'All collection items must start at the same column' +const startColMsg = 'All mapping items must start at the same column' export function resolveBlockMap( doc: Document.Parsed, @@ -32,8 +32,15 @@ export function resolveBlockMap( ) const implicitKey = keyProps.found === -1 if (implicitKey) { - if (key && 'indent' in key && key.indent !== indent) - onError(offset, startColMsg) + if (key) { + if (key.type === 'block-seq') + onError( + offset, + 'A block sequence may not be used as an implicit map key' + ) + else if ('indent' in key && key.indent !== indent) + onError(offset, startColMsg) + } if (keyProps.anchor || keyProps.tagName || sep) { const err = validateImplicitKey(key) if (err === 'single-line') diff --git a/src/compose/resolve-block-seq.ts b/src/compose/resolve-block-seq.ts index 1bfc6b14..2d76e52c 100644 --- a/src/compose/resolve-block-seq.ts +++ b/src/compose/resolve-block-seq.ts @@ -15,7 +15,7 @@ export function resolveBlockSeq( const seq = new YAMLSeq(doc.schema) seq.type = Type.SEQ if (anchor) doc.anchors.setAnchor(seq, anchor) - for (const { start, value } of items) { + loop: for (const { start, value } of items) { const props = resolveProps( doc, start, @@ -27,7 +27,11 @@ export function resolveBlockSeq( offset += props.length if (props.found === -1) { if (props.anchor || props.tagName || value) { - onError(offset, 'Sequence item without - indicator') + const msg = + value && value.type === 'block-seq' + ? 'All sequence items must start at the same column' + : 'Sequence item without - indicator' + onError(offset, msg) } else { // TODO: assert being at last item? if (props.comment) seq.comment = props.comment diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index 0651a5ae..b7c99457 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -13,7 +13,6 @@ export function resolveFlowCollection( _anchor: string | null, onError: (offset: number, message: string, warning?: boolean) => void ) { - let offset = fc.offset const isMap = fc.start.source === '{' const coll = isMap ? new YAMLMap(doc.schema) : new YAMLSeq(doc.schema) coll.type = isMap ? Type.FLOW_MAP : Type.FLOW_SEQ @@ -30,6 +29,7 @@ export function resolveFlowCollection( let anchor = '' let tagName = '' + let offset = fc.offset + 1 let atExplicitKey = false let atValueEnd = false let nlAfterValueInSeq = false diff --git a/src/doc/directives.ts b/src/doc/directives.ts index 2f63c2fb..1807bbe5 100644 --- a/src/doc/directives.ts +++ b/src/doc/directives.ts @@ -127,8 +127,9 @@ export class Directives { } const [, handle, suffix] = source.match(/^(.*!)([^!]*)$/) as string[] + if (!suffix) onError(`The ${source} tag has no suffix`) const prefix = this.tags[handle] - if (prefix) return prefix + suffix + if (prefix) return prefix + decodeURIComponent(suffix) if (handle === '!') return source // local tag onError(`Could not resolve tag: ${source}`) diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 544783a5..812d7422 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -286,7 +286,8 @@ export class Lexer { } } this.indentValue = this.pushSpaces(false) - if (this.indentNext > this.indentValue) this.indentNext = this.indentValue + if (this.indentNext > this.indentValue && !isEmpty(this.charAt(1))) + this.indentNext = this.indentValue return this.parseBlockStart() } diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 769f36e2..7ba94233 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -260,7 +260,7 @@ export class Parser { step() { const top = this.peek(1) - if (this.type === 'doc-end' && top.type !== 'doc-end') { + if (this.type === 'doc-end' && (!top || top.type !== 'doc-end')) { while (this.stack.length > 0) this.pop() this.stack.push({ type: 'doc-end', @@ -317,8 +317,10 @@ export class Parser { break case 'block-map': { const it = top.items[top.items.length - 1] - if (it.value) top.items.push({ start: [], key: token, sep: [] }) - else if (it.sep) it.value = token + if (it.value) { + top.items.push({ start: [], key: token, sep: [] }) + this.onKeyLine = true + } else if (it.sep) it.value = token else { Object.assign(it, { key: token, sep: [] }) this.onKeyLine = true diff --git a/tests/doc/YAML-1.2.spec.js b/tests/doc/YAML-1.2.spec.js index faccc4ed..ce0d8c67 100644 --- a/tests/doc/YAML-1.2.spec.js +++ b/tests/doc/YAML-1.2.spec.js @@ -1187,7 +1187,8 @@ bar`, - !e! foo - !h!bar baz`, tgt: [['foo', 'baz']], - errors: [['Could not resolve tag: !h!bar', 'The !e! tag has no suffix.']] + errors: [['The !e! tag has no suffix', 'Could not resolve tag: !h!bar']], + warnings: [['Unresolved tag: tag:example,2000:app/']] }, 'Example 6.28. Non-Specific Tags': { diff --git a/tests/doc/errors.js b/tests/doc/errors.js index c2dab4ce..4f136dd4 100644 --- a/tests/doc/errors.js +++ b/tests/doc/errors.js @@ -68,7 +68,7 @@ describe('block collections', () => { const src = 'foo: "1"\n bar: 2\n' const doc = YAML.parseDocument(src) expect(doc.errors).toMatchObject([ - { message: 'All collection items must start at the same column' } + { message: 'All mapping items must start at the same column' } ]) expect(doc.contents).toMatchObject({ type: 'MAP', @@ -83,11 +83,11 @@ describe('block collections', () => { const src = '- "foo"\n - bar\n' const doc = YAML.parseDocument(src) expect(doc.errors).toMatchObject([ - { message: 'All collection items must start at the same column' } + { message: 'All sequence items must start at the same column' } ]) expect(doc.contents).toMatchObject({ type: 'SEQ', - items: [{ value: 'foo' }, { value: 'bar' }] + items: [{ value: 'foo' }, { items: [{ value: 'bar' }] }] }) }) @@ -95,7 +95,7 @@ describe('block collections', () => { const src = 'foo: "1"\n- bar\n' const doc = YAML.parseDocument(src) expect(doc.errors).toMatchObject([ - { message: 'A collection cannot be both a mapping and a sequence' }, + { message: 'A block sequence may not be used as an implicit map key' }, { message: 'Implicit keys need to be on a single line' }, { message: 'Implicit map keys need to be followed by map values' } ]) @@ -103,7 +103,7 @@ describe('block collections', () => { type: 'MAP', items: [ { key: { value: 'foo' }, value: { value: '1' } }, - { key: null, value: null } + { key: { items: [{ value: 'bar' }] }, value: null } ] }) }) From 70bb49c0634660220b8dcfa21f51ad43195cefd5 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 9 Jan 2021 14:30:57 +0200 Subject: [PATCH 64/89] Check implicit key lengths, refactoring also contains-newline check --- src/compose/resolve-block-map.ts | 24 ++++++++++--------- src/compose/resolve-flow-collection.ts | 11 ++++++--- src/compose/resolve-props.ts | 15 ++++++++++-- ...plicit-key.ts => util-contains-newline.ts} | 11 ++------- tests/doc/YAML-1.2.spec.js | 2 +- 5 files changed, 37 insertions(+), 26 deletions(-) rename src/compose/{validate-implicit-key.ts => util-contains-newline.ts} (75%) diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index 338ecec3..6bf96891 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -5,7 +5,7 @@ import type { BlockMap } from '../parse/parser.js' import { composeNode } from './compose-node.js' import { resolveMergePair } from './resolve-merge-pair.js' import { resolveProps } from './resolve-props.js' -import { validateImplicitKey } from './validate-implicit-key.js' +import { containsNewline } from './util-contains-newline.js' const startColMsg = 'All mapping items must start at the same column' @@ -41,14 +41,7 @@ export function resolveBlockMap( else if ('indent' in key && key.indent !== indent) onError(offset, startColMsg) } - if (keyProps.anchor || keyProps.tagName || sep) { - const err = validateImplicitKey(key) - if (err === 'single-line') - onError( - offset + keyProps.length, - 'Implicit keys need to be on a single line' - ) - } else { + if (!keyProps.anchor && !keyProps.tagName && !sep) { // TODO: assert being at last item? if (keyProps.comment) { if (map.comment) map.comment += '\n' + keyProps.comment @@ -58,6 +51,8 @@ export function resolveBlockMap( } } else if (keyProps.found !== indent) onError(offset, startColMsg) offset += keyProps.length + if (implicitKey && containsNewline(key)) + onError(offset, 'Implicit keys need to be on a single line') // key value const keyStart = offset @@ -76,8 +71,15 @@ export function resolveBlockMap( offset += valueProps.length if (valueProps.found !== -1) { - if (implicitKey && value?.type === 'block-map' && !valueProps.hasNewline) - onError(offset, 'Nested mappings are not allowed in compact mappings') + if (implicitKey) { + if (value?.type === 'block-map' && !valueProps.hasNewline) + onError(offset, 'Nested mappings are not allowed in compact mappings') + if (doc.options.strict && keyProps.start < valueProps.found - 1024) + onError( + offset, + 'The : indicator must be at most 1024 chars after the start of an implicit block mapping key' + ) + } // value value const valueNode = composeNode(doc, value || offset, valueProps, onError) offset = valueNode.range[1] diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index b7c99457..750083a3 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -5,7 +5,7 @@ import type { FlowCollection, SourceToken, Token } from '../parse/parser.js' import { composeNode } from './compose-node.js' import { resolveEnd } from './resolve-end.js' import { resolveMergePair } from './resolve-merge-pair.js' -import { validateImplicitKey } from './validate-implicit-key.js' +import { containsNewline } from './util-contains-newline.js' export function resolveFlowCollection( doc: Document.Parsed, @@ -141,8 +141,13 @@ export function resolveFlowCollection( 'Implicit keys of flow sequence pairs need to be on a single line' if (nlAfterValueInSeq) onError(offset, slMsg) else if (seqKeyToken) { - const err = validateImplicitKey(seqKeyToken) - if (err === 'single-line') onError(offset, slMsg) + if (containsNewline(seqKeyToken)) onError(offset, slMsg) + const start = 'offset' in seqKeyToken && seqKeyToken.offset + if (typeof start === 'number' && start < offset - 1024) + onError( + offset, + 'The : indicator must be at most 1024 chars after the start of an implicit flow sequence key' + ) seqKeyToken = null } } diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index 91040123..65507ffd 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -31,6 +31,7 @@ export function resolveProps( let anchor = '' let tagName = '' let found = -1 + let start: number | null = null for (const token of tokens) { switch (token.type) { case 'space': @@ -65,6 +66,7 @@ export function resolveProps( if (anchor) onError(offset + length, 'A node can have at most one anchor') anchor = token.source.substring(1) + if (start === null) start = offset + length atNewline = false hasSpace = false break @@ -74,6 +76,7 @@ export function resolveProps( onError(offset, msg) ) if (tn) tagName = tn + if (start === null) start = offset + length atNewline = false hasSpace = false break @@ -91,6 +94,14 @@ export function resolveProps( } if (token.source) length += token.source.length } - if (!comment && isSpaceBefore(sep)) spaceBefore = true - return { found, spaceBefore, comment, hasNewline, anchor, tagName, length } + return { + found, + spaceBefore: spaceBefore || (!comment && isSpaceBefore(sep)), + comment, + hasNewline, + anchor, + tagName, + length, + start: start ?? offset + length + } } diff --git a/src/compose/validate-implicit-key.ts b/src/compose/util-contains-newline.ts similarity index 75% rename from src/compose/validate-implicit-key.ts rename to src/compose/util-contains-newline.ts index 08caa0e9..10a08c07 100644 --- a/src/compose/validate-implicit-key.ts +++ b/src/compose/util-contains-newline.ts @@ -1,6 +1,7 @@ import type { Token } from '../parse/parser.js' -function containsNewline(key: Token) { +export function containsNewline(key: Token | null | undefined) { + if (!key) return null switch (key.type) { case 'alias': case 'scalar': @@ -26,11 +27,3 @@ function containsNewline(key: Token) { return true } } - -export function validateImplicitKey(key: Token | null | undefined) { - if (key) { - if (containsNewline(key)) return 'single-line' - // TODO: check 1024 chars - } - return null -} diff --git a/tests/doc/YAML-1.2.spec.js b/tests/doc/YAML-1.2.spec.js index ce0d8c67..d5d35bfe 100644 --- a/tests/doc/YAML-1.2.spec.js +++ b/tests/doc/YAML-1.2.spec.js @@ -1467,7 +1467,7 @@ foo: bar errors: [ [ 'Implicit keys of flow sequence pairs need to be on a single line', - 'The "foo xxxx...xxxx bar" key is too long' + 'The : indicator must be at most 1024 chars after the start of an implicit flow sequence key' ] ] } From 2d63b15a2cf478981f4e84e7bccdb00f7c1e88bc Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 9 Jan 2021 16:42:19 +0200 Subject: [PATCH 65/89] Improve null value printing Now either the collection needs to be a Set (!!set), or have a truly empty value in order to print as `? foo` rather than `foo: null`. This is necessary, as more empty strings are now parsed as `Scalar` rather than just `null`. --- src/ast/Collection.js | 20 +++++--------------- src/ast/Pair.js | 6 +++++- src/ast/YAMLMap.js | 2 ++ src/ast/index.d.ts | 4 ++-- src/compose/compose-scalar.ts | 19 ++++++++++--------- src/tags/core.js | 2 +- src/tags/yaml-1.1/index.js | 2 +- src/tags/yaml-1.1/set.js | 10 +++++++--- tests/doc/parse.js | 12 ++++++------ tests/doc/stringify.js | 14 ++++++++------ types.d.ts | 4 ++-- 11 files changed, 49 insertions(+), 46 deletions(-) diff --git a/src/ast/Collection.js b/src/ast/Collection.js index 92100185..c7371275 100644 --- a/src/ast/Collection.js +++ b/src/ast/Collection.js @@ -88,13 +88,14 @@ export class Collection extends Node { : undefined } - hasAllNullValues() { + hasAllNullValues(allowScalar) { return this.items.every(node => { if (!node || node.type !== 'PAIR') return false const n = node.value return ( n == null || - (n instanceof Scalar && + (allowScalar && + n instanceof Scalar && n.value == null && !n.commentBefore && !n.comment && @@ -129,23 +130,12 @@ export class Collection extends Node { return null } - toString( - ctx, - { blockItem, flowChars, isMap, itemIndent }, - onComment, - onChompKeep - ) { + toString(ctx, { blockItem, flowChars, itemIndent }, onComment, onChompKeep) { const { indent, indentStep, stringify } = ctx const inFlow = this.type === Type.FLOW_MAP || this.type === Type.FLOW_SEQ || ctx.inFlow if (inFlow) itemIndent += indentStep - const allNullValues = isMap && this.hasAllNullValues() - ctx = Object.assign({}, ctx, { - allNullValues, - indent: itemIndent, - inFlow, - type: null - }) + ctx = Object.assign({}, ctx, { indent: itemIndent, inFlow, type: null }) let chompKeep = false let hasItemWithNewLine = false const nodes = this.items.reduce((nodes, item, i) => { diff --git a/src/ast/Pair.js b/src/ast/Pair.js index 28cdb148..cd4e5855 100644 --- a/src/ast/Pair.js +++ b/src/ast/Pair.js @@ -121,6 +121,7 @@ export class Pair extends Node { : typeof key === 'object')) const { allNullValues, doc, indent, indentStep, stringify } = ctx ctx = Object.assign({}, ctx, { + allNullValues: false, implicitKey: !explicitKey && (simpleKeys || !allNullValues), indent: indent + indentStep }) @@ -139,7 +140,10 @@ export class Pair extends Node { ) explicitKey = true } - if (allNullValues && !simpleKeys) { + if ( + (allNullValues || (value == null && (explicitKey || ctx.inFlow))) && + !simpleKeys + ) { if (this.comment) { str = addComment(str, ctx.indent, this.comment) if (onComment) onComment() diff --git a/src/ast/YAMLMap.js b/src/ast/YAMLMap.js index 0d877f02..4d2f2553 100644 --- a/src/ast/YAMLMap.js +++ b/src/ast/YAMLMap.js @@ -80,6 +80,8 @@ export class YAMLMap extends Collection { `Map items must all be pairs; found ${JSON.stringify(item)} instead` ) } + if (!ctx.allNullValues && this.hasAllNullValues(false)) + ctx = Object.assign({}, ctx, { allNullValues: true }) return super.toString( ctx, { diff --git a/src/ast/index.d.ts b/src/ast/index.d.ts index 8312e630..370732b4 100644 --- a/src/ast/index.d.ts +++ b/src/ast/index.d.ts @@ -140,13 +140,14 @@ export class Collection extends Node { */ set(key: any, value: any): void setIn(path: Iterable, value: any): void + + hasAllNullValues(allowScalar?: boolean): boolean } export class YAMLMap extends Collection { static readonly tagName: 'tag:yaml.org,2002:map' type?: Type.FLOW_MAP | Type.MAP items: Array - hasAllNullValues(): boolean toJSON(arg?: any, ctx?: AST.NodeToJsonContext): object | Map toString( ctx?: Schema.StringifyContext, @@ -168,7 +169,6 @@ export class YAMLSeq extends Collection { get(key: number | string | Scalar, keepScalar?: boolean): any has(key: number | string | Scalar): boolean set(key: number | string | Scalar, value: any): void - hasAllNullValues(): boolean toJSON(arg?: any, ctx?: AST.NodeToJsonContext): any[] toString( ctx?: Schema.StringifyContext, diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index b361fb96..de2e6d36 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -18,10 +18,9 @@ export function composeScalar( ? resolveBlockScalar(token, doc.options.strict, onError) : resolveFlowScalar(token, doc.options.strict, onError) - const tag = - findScalarTagByName(doc.schema, value, tagName, onError) || - findScalarTagByTest(doc.schema, value, token.type === 'scalar') || - findScalarTagByName(doc.schema, value, '!', onError) + const tag = tagName + ? findScalarTagByName(doc.schema, value, tagName, onError) + : findScalarTagByTest(doc.schema, value, token.type === 'scalar') let scalar: Scalar try { @@ -41,14 +40,16 @@ export function composeScalar( return scalar as Scalar.Parsed } +const defaultScalarTag = (schema: Schema) => + schema.tags.find(tag => tag.tag === 'tag:yaml.org,2002:str') + function findScalarTagByName( schema: Schema, value: string, - tagName: string | null, + tagName: string, onError: (offset: number, message: string, warning?: boolean) => void ) { - if (!tagName) return null - if (tagName === '!') tagName = 'tag:yaml.org,2002:str' // non-specific tag + if (tagName === '!') return defaultScalarTag(schema) // non-specific tag const matchWithTest: Schema.Tag[] = [] for (const tag of schema.tags) { if (tag.tag === tagName) { @@ -65,7 +66,7 @@ function findScalarTagByName( return kt } onError(0, `Unresolved tag: ${tagName}`, tagName !== 'tag:yaml.org,2002:str') - return null + return defaultScalarTag(schema) } function findScalarTagByTest(schema: Schema, value: string, apply: boolean) { @@ -74,5 +75,5 @@ function findScalarTagByTest(schema: Schema, value: string, apply: boolean) { if (tag.default && tag.test?.test(value)) return tag } } - return null + return defaultScalarTag(schema) } diff --git a/src/tags/core.js b/src/tags/core.js index df9a16a5..33cb59fb 100644 --- a/src/tags/core.js +++ b/src/tags/core.js @@ -36,7 +36,7 @@ export const nullObj = { test: /^(?:~|[Nn]ull|NULL)?$/, resolve: str => { const node = new Scalar(null) - node.sourceStr = str + if (str) node.sourceStr = str return node }, options: nullOptions, diff --git a/src/tags/yaml-1.1/index.js b/src/tags/yaml-1.1/index.js index bb50df63..2781a693 100644 --- a/src/tags/yaml-1.1/index.js +++ b/src/tags/yaml-1.1/index.js @@ -88,7 +88,7 @@ export const yaml11 = failsafe.concat( test: /^(?:~|[Nn]ull|NULL)?$/, resolve: str => { const node = new Scalar(null) - node.sourceStr = str + if (str) node.sourceStr = str return node }, options: nullOptions, diff --git a/src/tags/yaml-1.1/set.js b/src/tags/yaml-1.1/set.js index f421ad34..f27a8297 100644 --- a/src/tags/yaml-1.1/set.js +++ b/src/tags/yaml-1.1/set.js @@ -44,15 +44,19 @@ export class YAMLSet extends YAMLMap { toString(ctx, onComment, onChompKeep) { if (!ctx) return JSON.stringify(this) - if (this.hasAllNullValues()) - return super.toString(ctx, onComment, onChompKeep) + if (this.hasAllNullValues(true)) + return super.toString( + Object.assign({}, ctx, { allNullValues: true }), + onComment, + onChompKeep + ) else throw new Error('Set items must all have null values') } } function parseSet(map, onError) { if (map instanceof YAMLMap) { - if (map.hasAllNullValues()) return Object.assign(new YAMLSet(), map) + if (map.hasAllNullValues(true)) return Object.assign(new YAMLSet(), map) else onError('Set items must all have null values') } else onError('Expected a mapping for this tag') return map diff --git a/tests/doc/parse.js b/tests/doc/parse.js index 46dc8c66..ce03b4f6 100644 --- a/tests/doc/parse.js +++ b/tests/doc/parse.js @@ -465,18 +465,18 @@ describe('maps with no values', () => { test('block map', () => { const src = `a: null\n? b #c` const doc = YAML.parseDocument(src) - expect(String(doc)).toBe(`? a\n? b #c\n`) - doc.contents.items[1].value = 'x' - expect(String(doc)).toBe(`a: null\n? b #c\n: x\n`) + expect(String(doc)).toBe(`a: null\nb: #c\n`) + doc.set('b', 'x') + expect(String(doc)).toBe(`a: null\nb: #c\n x\n`) }) test('flow map', () => { const src = `{\na: null,\n? b\n}` const doc = YAML.parseDocument(src) - expect(String(doc)).toBe(`{ a, b }\n`) + expect(String(doc)).toBe(`{ a: null, b }\n`) doc.contents.items[1].comment = 'c' - expect(String(doc)).toBe(`{\n a,\n b #c\n}\n`) - doc.contents.items[1].value = 'x' + expect(String(doc)).toBe(`{\n a: null,\n b #c\n}\n`) + doc.set('b', 'x') expect(String(doc)).toBe(`{\n a: null,\n b: #c\n x\n}\n`) }) }) diff --git a/tests/doc/stringify.js b/tests/doc/stringify.js index 29aee39f..91e6a842 100644 --- a/tests/doc/stringify.js +++ b/tests/doc/stringify.js @@ -284,7 +284,7 @@ foo: test('do not match nulls', () => { const set = { a: null, b: null } - expect(YAML.stringify(set)).toBe('? a\n? b\n') + expect(YAML.stringify(set)).toBe('a: null\nb: null\n') }) }) @@ -327,7 +327,7 @@ z: test('Document as key', () => { const doc = new YAML.Document({ a: 1 }) doc.add(new YAML.Document({ b: 2, c: 3 })) - expect(String(doc)).toBe('a: 1\n? b: 2\n c: 3\n: null\n') + expect(String(doc)).toBe('a: 1\n? b: 2\n c: 3\n') }) }) @@ -593,11 +593,11 @@ describe('scalar styles', () => { }) describe('simple keys', () => { - test('key with null value', () => { - const doc = YAML.parseDocument('~: ~') + test('key with no value', () => { + const doc = YAML.parseDocument('? ~') expect(String(doc)).toBe('? ~\n') doc.options.simpleKeys = true - expect(String(doc)).toBe('~: ~\n') + expect(String(doc)).toBe('~: null\n') }) test('key with block scalar value', () => { @@ -828,7 +828,9 @@ describe('Scalar options', () => { test('Use defaultType for explicit keys', () => { YAML.scalarOptions.str.defaultType = Type.QUOTE_DOUBLE YAML.scalarOptions.str.defaultKeyType = Type.QUOTE_SINGLE - expect(YAML.stringify({ foo: null })).toBe('? "foo"\n') + const doc = new YAML.Document({ foo: null }) + doc.contents.items[0].value = null + expect(String(doc)).toBe('? "foo"\n') }) }) diff --git a/types.d.ts b/types.d.ts index 0d7737f3..5727ecb2 100644 --- a/types.d.ts +++ b/types.d.ts @@ -290,12 +290,13 @@ export class Collection extends Node { */ set(key: any, value: any): void setIn(path: Iterable, value: any): void + + hasAllNullValues(allowScalar?: boolean): boolean } export class YAMLMap extends Collection { type?: Type.FLOW_MAP | Type.MAP items: Array - hasAllNullValues(): boolean toJSON(arg?: any, ctx?: AST.NodeToJsonContext): object | Map toString( ctx?: Schema.StringifyContext, @@ -310,7 +311,6 @@ export class YAMLSeq extends Collection { get(key: number | string | Scalar, keepScalar?: boolean): any has(key: number | string | Scalar): boolean set(key: number | string | Scalar, value: any): void - hasAllNullValues(): boolean toJSON(arg?: any, ctx?: AST.NodeToJsonContext): any[] toString( ctx?: Schema.StringifyContext, From 0f5bf2006b7519a8d5a5dd48c8e636ad319c39d4 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 9 Jan 2021 22:50:38 +0200 Subject: [PATCH 66/89] Bugfixes, mostly for comment & whitespace handling --- src/ast/Pair.js | 37 ++++++++++++++++++++------ src/compose/resolve-block-map.ts | 4 +-- src/compose/resolve-flow-collection.ts | 14 +++++++--- src/compose/resolve-props.ts | 16 +++-------- src/parse/lexer.ts | 20 ++++++++++---- src/parse/parser.ts | 36 ++++++++++++++++++------- tests/doc/parse.js | 2 +- tests/doc/stringify.js | 2 +- 8 files changed, 88 insertions(+), 43 deletions(-) diff --git a/src/ast/Pair.js b/src/ast/Pair.js index cd4e5855..08fee979 100644 --- a/src/ast/Pair.js +++ b/src/ast/Pair.js @@ -69,6 +69,20 @@ export class Pair extends Node { } } + get spaceBefore() { + return this.key instanceof Node ? this.key.spaceBefore : undefined + } + + set spaceBefore(sb) { + if (this.key == null) this.key = new Scalar(null) + if (this.key instanceof Node) this.key.spaceBefore = sb + else { + const msg = + 'Pair.spaceBefore is an alias for Pair.key.spaceBefore. To set it, the key must be a Node.' + throw new Error(msg) + } + } + addToJSMap(ctx, map) { const key = toJS(this.key, '', ctx) if (map instanceof Map) { @@ -113,7 +127,7 @@ export class Pair extends Node { let explicitKey = !simpleKeys && (!key || - keyComment || + (keyComment && value == null) || (key instanceof Node ? key instanceof Collection || key.type === Type.BLOCK_FOLDED || @@ -132,30 +146,37 @@ export class Pair extends Node { () => (keyComment = null), () => (chompKeep = true) ) - str = addComment(str, ctx.indent, keyComment) - if (!explicitKey && str.length > 1024) { + if (!explicitKey && !ctx.inFlow && str.length > 1024) { if (simpleKeys) throw new Error( 'With simple keys, single line scalar must not span more than 1024 characters' ) explicitKey = true } + if ( - (allNullValues || (value == null && (explicitKey || ctx.inFlow))) && - !simpleKeys + (allNullValues && (!simpleKeys || ctx.inFlow)) || + (value == null && (explicitKey || ctx.inFlow)) ) { + str = addComment(str, ctx.indent, keyComment) if (this.comment) { + if (keyComment) str += '\n' str = addComment(str, ctx.indent, this.comment) if (onComment) onComment() } else if (chompKeep && !keyComment && onChompKeep) onChompKeep() return ctx.inFlow && !explicitKey ? str : `? ${str}` } - str = explicitKey ? `? ${str}\n${indent}:` : `${str}:` + + str = explicitKey + ? `? ${addComment(str, ctx.indent, keyComment)}\n${indent}:` + : addComment(`${str}:`, ctx.indent, keyComment) if (this.comment) { + if (keyComment && !explicitKey) str += '\n' // expected (but not strictly required) to be a single-line comment str = addComment(str, ctx.indent, this.comment) if (onComment) onComment() } + let vcb = '' let valueComment = null if (value instanceof Node) { @@ -169,7 +190,7 @@ export class Pair extends Node { value = doc.createNode(value) } ctx.implicitKey = false - if (!explicitKey && !this.comment && value instanceof Scalar) + if (!explicitKey && !keyComment && !this.comment && value instanceof Scalar) ctx.indentAtStart = str.length + 1 chompKeep = false if ( @@ -192,7 +213,7 @@ export class Pair extends Node { () => (chompKeep = true) ) let ws = ' ' - if (vcb || this.comment) { + if (vcb || keyComment || this.comment) { ws = `${vcb}\n${ctx.indent}` } else if (!explicitKey && value instanceof Collection) { const flow = valueStr[0] === '[' || valueStr[0] === '{' diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index 6bf96891..9c53b76c 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -90,8 +90,8 @@ export function resolveBlockMap( if (implicitKey) onError(keyStart, 'Implicit map keys need to be followed by map values') if (valueProps.comment) { - if (map.comment) map.comment += '\n' + valueProps.comment - else map.comment = valueProps.comment + if (keyNode.comment) keyNode.comment += '\n' + valueProps.comment + else keyNode.comment = valueProps.comment } map.items.push(new Pair(keyNode)) } diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index 750083a3..a460c3e6 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -30,6 +30,7 @@ export function resolveFlowCollection( let tagName = '' let offset = fc.offset + 1 + let atLineStart = false let atExplicitKey = false let atValueEnd = false let nlAfterValueInSeq = false @@ -80,14 +81,14 @@ export function resolveFlowCollection( 'Comments must be separated from other tokens by white space characters' ) const cb = token.source.substring(1) - if (!hasComment) { - if (newlines) spaceBefore = true - comment = cb - } else comment += newlines + cb + if (!hasComment) comment = cb + else comment += newlines + cb + atLineStart = false hasComment = true newlines = '' break case 'newline': + if (atLineStart && !hasComment) spaceBefore = true if (atValueEnd) { if (hasComment) { let node = coll.items[coll.items.length - 1] @@ -102,11 +103,13 @@ export function resolveFlowCollection( newlines += token.source if (!isMap && !key && value) nlAfterValueInSeq = true } + atLineStart = true hasSpace = true break case 'anchor': if (anchor) onError(offset, 'A node can have at most one anchor') anchor = token.source.substring(1) + atLineStart = false atValueEnd = false hasSpace = false break @@ -114,6 +117,7 @@ export function resolveFlowCollection( if (tagName) onError(offset, 'A node can have at most one tag') const tn = doc.directives.tagName(token.source, m => onError(offset, m)) if (tn) tagName = tn + atLineStart = false atValueEnd = false hasSpace = false break @@ -122,6 +126,7 @@ export function resolveFlowCollection( if (anchor || tagName) onError(offset, 'Anchors and tags must be after the ? indicator') atExplicitKey = true + atLineStart = false atValueEnd = false hasSpace = false break @@ -183,6 +188,7 @@ export function resolveFlowCollection( if (!isMap && !key && !atExplicitKey) seqKeyToken = token value = composeNode(doc, token, getProps(), onError) offset = value.range[1] + atLineStart = false isSourceToken = false atValueEnd = false hasSpace = false diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index 65507ffd..2bbd2a87 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -1,13 +1,6 @@ import type { Document } from '../doc/Document.js' import { SourceToken } from '../parse/parser.js' -function isSpaceBefore(sep: string) { - if (!sep) return false - const first = sep.indexOf('\n') - if (first === -1) return false - return sep.includes('\n', first + 1) -} - export function resolveProps( doc: Document.Parsed, tokens: SourceToken[], @@ -48,15 +41,14 @@ export function resolveProps( 'Comments must be separated from other tokens by white space characters' ) const cb = token.source.substring(1) - if (!hasComment) { - if (isSpaceBefore(sep)) spaceBefore = true - comment = cb - } else comment += sep + cb + if (!hasComment) comment = cb + else comment += sep + cb hasComment = true sep = '' break } case 'newline': + if (atNewline && !hasComment) spaceBefore = true atNewline = true hasNewline = true hasSpace = true @@ -96,7 +88,7 @@ export function resolveProps( } return { found, - spaceBefore: spaceBefore || (!comment && isSpaceBefore(sep)), + spaceBefore, comment, hasNewline, anchor, diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 812d7422..5d31ee1d 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -501,27 +501,37 @@ export class Lexer { parsePlainScalar() { const inFlow = this.flowLevel > 0 + let end = this.pos - 1 let i = this.pos - 1 let ch: string while ((ch = this.buffer[++i])) { if (ch === ':') { const next = this.buffer[i + 1] if (isEmpty(next) || (inFlow && next === ',')) break + end = i } else if (isEmpty(ch)) { const next = this.buffer[i + 1] if (next === '#' || (inFlow && invalidFlowScalarChars.includes(next))) break - if (ch === '\n' || (ch === '\r' && next === '\n')) { - const ls = i + (ch === '\n' ? 1 : 2) - const cs = this.continueScalar(ls) + if (ch === '\r') { + if (next === '\n') { + i += 1 + ch = '\n' + } else end = i + } + if (ch === '\n') { + const cs = this.continueScalar(i + 1) if (cs === -1) break i = Math.max(i, cs - 2) // to advance, but still account for ' #' } - } else if (inFlow && invalidFlowScalarChars.includes(ch)) break + } else { + if (inFlow && invalidFlowScalarChars.includes(ch)) break + end = i + } } if (!ch && !this.atEnd) return this.setNext('plain-scalar') this.push(SCALAR) - this.pushToIndex(i, true) + this.pushToIndex(end + 1, true) return inFlow ? 'flow' : 'doc' } diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 7ba94233..700de7be 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -98,6 +98,20 @@ function includesToken(list: SourceToken[], type: SourceToken['type']) { return false } +function includesNonEmpty(list: SourceToken[]) { + for (let i = 0; i < list.length; ++i) { + switch (list[i].type) { + case 'space': + case 'comment': + case 'newline': + break + default: + return true + } + } + return false +} + function isFlowToken( token: Token | null ): token is FlowScalar | FlowCollection { @@ -320,8 +334,9 @@ export class Parser { if (it.value) { top.items.push({ start: [], key: token, sep: [] }) this.onKeyLine = true - } else if (it.sep) it.value = token - else { + } else if (it.sep) { + it.value = token + } else { Object.assign(it, { key: token, sep: [] }) this.onKeyLine = true } @@ -378,11 +393,7 @@ export class Parser { if (doc.value) return this.lineEnd(doc) switch (this.type) { case 'doc-start': { - const hasContent = - includesToken(doc.start, 'doc-start') || - includesToken(doc.start, 'anchor') || - includesToken(doc.start, 'tag') - if (hasContent) { + if (includesNonEmpty(doc.start)) { this.pop() this.step() } else doc.start.push(this.sourceToken) @@ -473,7 +484,10 @@ export class Parser { return } if (this.indent >= map.indent) { - const atNextItem = !this.onKeyLine && this.indent === map.indent + const atNextItem = + !this.onKeyLine && + this.indent === map.indent && + (it.sep || includesNonEmpty(it.start)) switch (this.type) { case 'anchor': case 'tag': @@ -629,8 +643,10 @@ export class Parser { } else { const parent = this.peek(2) if ( - (this.type === 'newline' || this.type == 'map-value-ind') && - parent.type === 'block-map' + parent.type === 'block-map' && + (this.type == 'map-value-ind' || + (this.type === 'newline' && + !parent.items[parent.items.length - 1].sep)) ) { this.pop() this.step() diff --git a/tests/doc/parse.js b/tests/doc/parse.js index ce03b4f6..073bb829 100644 --- a/tests/doc/parse.js +++ b/tests/doc/parse.js @@ -465,7 +465,7 @@ describe('maps with no values', () => { test('block map', () => { const src = `a: null\n? b #c` const doc = YAML.parseDocument(src) - expect(String(doc)).toBe(`a: null\nb: #c\n`) + expect(String(doc)).toBe(`a: null\n? b #c\n`) doc.set('b', 'x') expect(String(doc)).toBe(`a: null\nb: #c\n x\n`) }) diff --git a/tests/doc/stringify.js b/tests/doc/stringify.js index 91e6a842..fe85cdcc 100644 --- a/tests/doc/stringify.js +++ b/tests/doc/stringify.js @@ -611,7 +611,7 @@ describe('simple keys', () => { test('key with comment', () => { const doc = YAML.parseDocument('foo: bar') doc.contents.items[0].key.comment = 'FOO' - expect(String(doc)).toBe('? foo #FOO\n: bar\n') + expect(String(doc)).toBe('foo: #FOO\n bar\n') doc.options.simpleKeys = true expect(() => String(doc)).toThrow( /With simple keys, key nodes cannot have comments/ From 5e8ad7e77d8c409a77481edc985a4d7e00d2918c Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 10 Jan 2021 13:37:30 +0200 Subject: [PATCH 67/89] Fix document comment parsing; add directives to EmptyStream --- src/compose/compose-stream.ts | 109 ++++++++++++++++++++++------------ tests/doc/YAML-1.2.spec.js | 4 +- 2 files changed, 72 insertions(+), 41 deletions(-) diff --git a/src/compose/compose-stream.ts b/src/compose/compose-stream.ts index 9e833629..58182304 100644 --- a/src/compose/compose-stream.ts +++ b/src/compose/compose-stream.ts @@ -6,23 +6,51 @@ import { Parser } from '../parse/parser.js' import { composeDoc } from './compose-doc.js' import { resolveEnd } from './resolve-end.js' +function parsePrelude(prelude: string[]) { + let comment = '' + let atComment = false + let afterEmptyLine = false + for (let i = 0; i < prelude.length; ++i) { + const source = prelude[i] + switch (source[0]) { + case '#': + comment += + (comment === '' ? '' : afterEmptyLine ? '\n\n' : '\n') + + source.substring(1) + atComment = true + afterEmptyLine = false + break + case '%': + if (prelude[i + 1][0] !== '#') i += 1 + atComment = false + break + default: + // This may be wrong after doc-end, but in that case it doesn't matter + if (!atComment) afterEmptyLine = true + atComment = false + } + } + return { comment, afterEmptyLine } +} + export interface EmptyStream extends Array { empty: true comment: string + directives: Directives errors: YAMLParseError[] warnings: YAMLWarning[] } /** * @returns If an empty `docs` array is returned, it will be of type - * EmptyStream. In TypeScript, you may use `'empty' in docs` as a - * type guard for it, as `docs.length === 0` won't catch it. + * EmptyStream. In TypeScript, you should use `'empty' in docs` as + * a type guard for it. */ export function composeStream( source: string, forceDoc: boolean, options?: Options -) { +): Document.Parsed[] | EmptyStream { const directives = new Directives({ version: options?.version || defaultOptions.version || '1.2' }) @@ -30,7 +58,7 @@ export function composeStream( const lines: number[] = [] let atDirectives = false - let comment = '' + let prelude: string[] = [] let errors: YAMLParseError[] = [] let warnings: YAMLWarning[] = [] const onError = (offset: number, message: string, warning?: boolean) => { @@ -38,14 +66,32 @@ export function composeStream( ? warnings.push(new YAMLWarning(offset, message)) : errors.push(new YAMLParseError(offset, message)) } - const decorate = (doc: Document.Parsed) => { - if (comment) doc.commentBefore = comment.trimRight() - comment = '' - doc.errors = errors - errors = [] + const decorate = (doc: Document.Parsed, afterDoc: boolean) => { + const { comment, afterEmptyLine } = parsePrelude(prelude) + //console.log({ dc: doc.comment, prelude, comment }) + if (comment) { + if (afterDoc) { + const dc = doc.comment + doc.comment = dc ? `${dc}\n${comment}` : comment + } else if (afterEmptyLine || doc.directivesEndMarker || !doc.contents) { + doc.commentBefore = comment + } else { + const cb = doc.contents.commentBefore + doc.contents.commentBefore = cb ? `${comment}\n${cb}` : comment + } + } - doc.warnings = warnings + if (afterDoc) { + Array.prototype.push.apply(doc.errors, errors) + Array.prototype.push.apply(doc.warnings, warnings) + } else { + doc.errors = errors + doc.warnings = warnings + } + + prelude = [] + errors = [] warnings = [] } @@ -55,11 +101,12 @@ export function composeStream( switch (token.type) { case 'directive': directives.add(token.source, onError) + prelude.push(token.source) atDirectives = true break case 'document': { const doc = composeDoc(options, directives, token, onError) - decorate(doc) + decorate(doc, false) docs.push(doc) atDirectives = false break @@ -68,10 +115,8 @@ export function composeStream( case 'space': break case 'comment': - comment += token.source.substring(1) - break case 'newline': - if (comment) comment += token.source + prelude.push(token.source) break case 'error': { const msg = token.source @@ -95,13 +140,10 @@ export function composeStream( doc.options.strict, onError ) + decorate(doc, true) if (end.comment) { - if (doc.comment) doc.comment += `\n${end.comment}` - else doc.comment = end.comment - } - if (errors.length > 0) { - Array.prototype.push.apply(doc.errors, errors) - errors = [] + const dc = doc.comment + doc.comment = dc ? `${dc}\n${end.comment}` : end.comment } break } @@ -117,34 +159,23 @@ export function composeStream( if (forceDoc) { const doc = new Document(undefined, options) as Document.Parsed doc.directives = directives.atDocument() - if (atDirectives) { - const errMsg = 'Missing directives-end indicator line' - doc.errors.push(new YAMLParseError(source.length, errMsg)) - } + if (atDirectives) + onError(source.length, 'Missing directives-end indicator line') doc.setSchema() // FIXME: always do this in the constructor doc.range = [0, source.length] - docs.push(doc) + decorate(doc, false) + return [doc] } else { + const { comment } = parsePrelude(prelude) const empty: EmptyStream = Object.assign( [], { empty: true } as { empty: true }, - { comment: comment.trimRight(), errors, warnings } + { comment, directives, errors, warnings } ) return empty } } - if (comment || errors.length > 0 || warnings.length > 0) { - const lastDoc = docs[docs.length - 1] - comment = comment.trimRight() - if (comment) { - if (lastDoc.comment) lastDoc.comment += `\n${comment}` - else lastDoc.comment = comment - } - Array.prototype.push.apply(lastDoc.errors, errors) - Array.prototype.push.apply(lastDoc.warnings, warnings) - } - - // TS would complain without the cast, but docs is always non-empty here - return (docs as unknown) as [Document.Parsed, ...Document.Parsed[]] + decorate(docs[docs.length - 1], true) + return docs } diff --git a/tests/doc/YAML-1.2.spec.js b/tests/doc/YAML-1.2.spec.js index d5d35bfe..8f484294 100644 --- a/tests/doc/YAML-1.2.spec.js +++ b/tests/doc/YAML-1.2.spec.js @@ -713,7 +713,7 @@ mapping: { sky: blue, sea: green }`, tgt: [], special(src) { const doc = YAML.parseDocument(src) - expect(doc.comment).toBe(' Comment only.') + expect(doc.commentBefore).toBe(' Comment only.') } }, @@ -943,7 +943,7 @@ Chomping: | tgt: [], special(src) { const doc = YAML.parseDocument(src) - expect(doc.comment).toBe(' Comment') + expect(doc.commentBefore).toBe(' Comment') } }, From 8587841fc9f21ac1e26008af36c1ed59084bafdb Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 24 Jan 2021 12:02:39 +0200 Subject: [PATCH 68/89] Assign collection comments separated by empty line to preceding node --- src/ast/Pair.js | 11 ++--- src/ast/index.d.ts | 1 + src/compose/compose-stream.ts | 19 ++++++--- src/parse/parser.ts | 52 ++++++++++++++++++++++- tests/doc/comments.js | 80 ++++++++++++++++++++--------------- 5 files changed, 117 insertions(+), 46 deletions(-) diff --git a/src/ast/Pair.js b/src/ast/Pair.js index 08fee979..03f4e62e 100644 --- a/src/ast/Pair.js +++ b/src/ast/Pair.js @@ -160,8 +160,9 @@ export class Pair extends Node { ) { str = addComment(str, ctx.indent, keyComment) if (this.comment) { - if (keyComment) str += '\n' - str = addComment(str, ctx.indent, this.comment) + if (keyComment && !this.comment.includes('\n')) + str += `\n${ctx.indent || ''}#${this.comment}` + else str = addComment(str, ctx.indent, this.comment) if (onComment) onComment() } else if (chompKeep && !keyComment && onChompKeep) onChompKeep() return ctx.inFlow && !explicitKey ? str : `? ${str}` @@ -171,9 +172,9 @@ export class Pair extends Node { ? `? ${addComment(str, ctx.indent, keyComment)}\n${indent}:` : addComment(`${str}:`, ctx.indent, keyComment) if (this.comment) { - if (keyComment && !explicitKey) str += '\n' - // expected (but not strictly required) to be a single-line comment - str = addComment(str, ctx.indent, this.comment) + if (keyComment && !explicitKey && !this.comment.includes('\n')) + str += `\n${ctx.indent || ''}#${this.comment}` + else str = addComment(str, ctx.indent, this.comment) if (onComment) onComment() } diff --git a/src/ast/index.d.ts b/src/ast/index.d.ts index 370732b4..f1f3931a 100644 --- a/src/ast/index.d.ts +++ b/src/ast/index.d.ts @@ -179,6 +179,7 @@ export class YAMLSeq extends Collection { export namespace YAMLSeq { interface Parsed extends YAMLSeq { + items: Node[] range: [number, number] } } diff --git a/src/compose/compose-stream.ts b/src/compose/compose-stream.ts index 58182304..9a7e01d5 100644 --- a/src/compose/compose-stream.ts +++ b/src/compose/compose-stream.ts @@ -1,3 +1,4 @@ +import { Collection, Node } from '../ast/index.js' import { Directives } from '../doc/directives.js' import { Document } from '../doc/Document.js' import { YAMLParseError, YAMLWarning } from '../errors.js' @@ -71,14 +72,22 @@ export function composeStream( const { comment, afterEmptyLine } = parsePrelude(prelude) //console.log({ dc: doc.comment, prelude, comment }) if (comment) { + const dc = doc.contents if (afterDoc) { - const dc = doc.comment - doc.comment = dc ? `${dc}\n${comment}` : comment - } else if (afterEmptyLine || doc.directivesEndMarker || !doc.contents) { + doc.comment = doc.comment ? `${doc.comment}\n${comment}` : comment + } else if (afterEmptyLine || doc.directivesEndMarker || !dc) { doc.commentBefore = comment + } else if ( + dc instanceof Collection && + (dc.type === 'MAP' || dc.type === 'SEQ') && + dc.items.length > 0 + ) { + const it = dc.items[0] as Node + const cb = it.commentBefore + it.commentBefore = cb ? `${comment}\n${cb}` : comment } else { - const cb = doc.contents.commentBefore - doc.contents.commentBefore = cb ? `${comment}\n${cb}` : comment + const cb = dc.commentBefore + dc.commentBefore = cb ? `${comment}\n${cb}` : comment } } diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 700de7be..d1f9bc43 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -112,6 +112,37 @@ function includesNonEmpty(list: SourceToken[]) { return false } +function atFirstEmptyLineAfterComments(start: SourceToken[]) { + let hasComment = false + for (let i = 0; i < start.length; ++i) { + switch (start[i].type) { + case 'space': + break + case 'comment': + hasComment = true + break + case 'newline': + if (!hasComment) return false + break + default: + return false + } + } + if (hasComment) { + for (let i = start.length - 1; i >= 0; --i) { + switch (start[i].type) { + case 'space': + break + case 'newline': + return true + default: + return false + } + } + } + return false +} + function isFlowToken( token: Token | null ): token is FlowScalar | FlowCollection { @@ -475,6 +506,15 @@ export class Parser { switch (this.type) { case 'newline': this.onKeyLine = false + if (!it.sep && atFirstEmptyLineAfterComments(it.start)) { + const prev = map.items[map.items.length - 2] + const end = (prev?.value as { end: SourceToken[] })?.end + if (Array.isArray(end)) { + Array.prototype.push.apply(end, it.start) + it.start = [this.sourceToken] + return + } + } // fallthrough case 'space': case 'comment': @@ -578,9 +618,19 @@ export class Parser { blockSequence(seq: BlockSequence) { const it = seq.items[seq.items.length - 1] switch (this.type) { + case 'newline': + if (!it.value && atFirstEmptyLineAfterComments(it.start)) { + const prev = seq.items[seq.items.length - 2] + const end = (prev?.value as { end: SourceToken[] })?.end + if (Array.isArray(end)) { + Array.prototype.push.apply(end, it.start) + it.start = [this.sourceToken] + return + } + } + // fallthrough case 'space': case 'comment': - case 'newline': if (it.value) seq.items.push({ start: [this.sourceToken] }) else it.start.push(this.sourceToken) return diff --git a/tests/doc/comments.js b/tests/doc/comments.js index 68e873a1..4ed749fa 100644 --- a/tests/doc/comments.js +++ b/tests/doc/comments.js @@ -52,7 +52,7 @@ describe('parse comments', () => { expect(doc.contents.comment).toBe('c1') expect(doc.comment).toBe('c2') expect(doc.contents.value).toBe('value') - expect(doc.contents.range).toMatchObject([4, 13]) + expect(doc.contents.range).toMatchObject([4, 14]) }) test('"quoted"', () => { @@ -62,7 +62,7 @@ describe('parse comments', () => { expect(doc.contents.comment).toBe('c1') expect(doc.comment).toBe('c2') expect(doc.contents.value).toBe('value') - expect(doc.contents.range).toMatchObject([4, 15]) + expect(doc.contents.range).toMatchObject([4, 16]) }) test('block', () => { @@ -89,8 +89,8 @@ describe('parse comments', () => { expect(doc).toMatchObject({ contents: { items: [ - { commentBefore: 'c0', range: [6, 13] }, - { commentBefore: 'c1' } + { commentBefore: 'c0', value: 'value 1', comment: 'c1' }, + { value: 'value 2' } ], range: [4, 29] }, @@ -112,7 +112,7 @@ describe('parse comments', () => { const doc = YAML.parseDocument(src) expect(doc).toMatchObject({ contents: { - items: [{}, { commentBefore: 'c0\nc1\nc2' }] + items: [{ comment: 'c0\nc1' }, { commentBefore: 'c2' }] }, comment: 'c3\nc4' }) @@ -131,7 +131,10 @@ key2: value 2 const doc = YAML.parseDocument(src) expect(doc).toMatchObject({ contents: { - items: [{ commentBefore: 'c0' }, { commentBefore: 'c1' }] + items: [ + { key: { commentBefore: 'c0' }, value: { comment: 'c1' } }, + { key: {}, value: {} } + ] }, comment: 'c2' }) @@ -150,7 +153,10 @@ key2: value 2 const doc = YAML.parseDocument(src) expect(doc).toMatchObject({ contents: { - items: [{}, { commentBefore: 'c0\nc1\nc2' }] + items: [ + { value: { comment: 'c0\nc1' } }, + { key: { commentBefore: 'c2' } } + ] }, comment: 'c3\nc4' }) @@ -168,17 +174,19 @@ key2: value 2 k3: v3 #c5\n` const doc = YAML.parseDocument(src) - expect(doc.contents.items).toMatchObject([ - { - commentBefore: 'c0\nc1', - items: [ - {}, - { commentBefore: 'c2', value: { comment: 'c3' } }, - { commentBefore: 'c4' } - ] - } - ]) - expect(doc.comment).toBe('c5') + expect(doc.contents).toMatchObject({ + items: [ + { + commentBefore: 'c0\nc1', + items: [ + {}, + { commentBefore: 'c2', value: { comment: 'c3' } }, + { commentBefore: 'c4' } + ] + } + ], + comment: 'c5' + }) expect(String(doc)).toBe(`#c0 #c1 - k1: v1 @@ -203,21 +211,23 @@ k2: - v3 #c4 #c5\n` const doc = YAML.parseDocument(src) - expect(doc.contents.items).toMatchObject([ - { - comment: 'c1', - key: { commentBefore: 'c0', value: 'k1' }, - value: { - items: [{ value: 'v1' }, { commentBefore: 'c2', value: 'v2' }], - comment: 'c3' + expect(doc.contents).toMatchObject({ + items: [ + { + key: { commentBefore: 'c0', value: 'k1' }, + value: { + commentBefore: 'c1', + items: [{ value: 'v1' }, { commentBefore: 'c2', value: 'v2' }], + comment: 'c3' + } + }, + { + key: { value: 'k2' }, + value: { items: [{ value: 'v3', comment: 'c4' }] } } - }, - { - key: { value: 'k2' }, - value: { items: [{ value: 'v3', comment: 'c4' }] } - } - ]) - expect(doc.comment).toBe('c5') + ], + comment: 'c5' + }) expect(String(doc)).toBe(`#c0 k1: #c1 - v1 @@ -352,8 +362,8 @@ describe('stringify comments', () => { seq.comment = 'c5' expect(String(doc)).toBe( `#c0 -? map #c1 -: #c2 +map: #c1 + #c2 #c3 - value 1 #c4 @@ -667,7 +677,7 @@ entryB: expect(String(doc)).toBe(`a: b #c\n\n#d\n`) }) - test('comment association by indentation', () => { + test.skip('comment association by indentation', () => { const src = ` a: - b #c From eebde3c478fd88ad957e853631554b863a7f41da Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 24 Jan 2021 12:51:27 +0200 Subject: [PATCH 69/89] Do not include trailing lines for strip & clip chomped block scalars --- src/parse/lexer.ts | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index 5d31ee1d..cc3344ce 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -112,6 +112,13 @@ export class Lexer { */ blockScalarIndent = -1 + /** + * Block scalars that include a + (keep) chomping indicator in their header + * include trailing empty lines, which are otherwise excluded from the + * scalar's contents. + */ + blockScalarKeep = false + /** Current input */ buffer = '' @@ -453,13 +460,14 @@ export class Lexer { } parseBlockScalarHeader() { + this.blockScalarIndent = -1 + this.blockScalarKeep = false let i = this.pos while (true) { const ch = this.buffer[++i] - if (ch === '-' || ch === '+') continue - const n = Number(ch) - this.blockScalarIndent = n > 0 ? n - 1 : -1 - break + if (ch === '+') this.blockScalarKeep = true + else if (ch > '0' && ch <= '9') this.blockScalarIndent = Number(ch) - 1 + else if (ch !== '-') break } return this.pushUntil(ch => isEmpty(ch) || ch === '#') } @@ -494,6 +502,16 @@ export class Lexer { nl = this.buffer.length } } + if (!this.blockScalarKeep) { + do { + let i = nl - 1 + let ch = this.buffer[i] + if (ch === '\r') ch = this.buffer[--i] + while (ch === ' ' || ch === '\t') ch = this.buffer[--i] + if (ch === '\n' && i >= this.pos) nl = i + else break + } while (true) + } this.push(SCALAR) this.pushToIndex(nl + 1, true) return this.parseLineStart() From 20f422076652bc8f866eef3aaf73d945cc9227d7 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 24 Jan 2021 14:16:55 +0200 Subject: [PATCH 70/89] Grab trailing less-indented comments from inner block map/seq to parent --- src/parse/parser.ts | 30 ++++++++++++++++++-- tests/doc/comments.js | 64 +++++++++++++++++++++++-------------------- 2 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/parse/parser.ts b/src/parse/parser.ts index d1f9bc43..0e718718 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -69,7 +69,7 @@ export interface BlockSequence { type: 'block-seq' offset: number indent: number - items: Array<{ start: SourceToken[]; value?: Token }> + items: Array<{ start: SourceToken[]; value?: Token; sep?: never }> } export interface FlowCollection { @@ -365,11 +365,13 @@ export class Parser { if (it.value) { top.items.push({ start: [], key: token, sep: [] }) this.onKeyLine = true + return } else if (it.sep) { it.value = token } else { Object.assign(it, { key: token, sep: [] }) this.onKeyLine = true + return } break } @@ -386,6 +388,30 @@ export class Parser { this.pop() this.pop(token) } + + if ( + (top.type === 'document' || + top.type === 'block-map' || + top.type === 'block-seq') && + (token.type === 'block-map' || token.type === 'block-seq') + ) { + const last = token.items[token.items.length - 1] + if ( + last && + !last.sep && + !last.value && + last.start.length > 0 && + !includesNonEmpty(last.start) && + (token.indent === 0 || + last.start.every( + st => st.type !== 'comment' || st.indent < token.indent + )) + ) { + if (top.type === 'document') top.end = last.start + else top.items.push({ start: last.start }) + token.items.splice(-1, 1) + } + } } } @@ -628,7 +654,7 @@ export class Parser { return } } - // fallthrough + // fallthrough case 'space': case 'comment': if (it.value) seq.items.push({ start: [this.sourceToken] }) diff --git a/tests/doc/comments.js b/tests/doc/comments.js index 4ed749fa..dc0bc785 100644 --- a/tests/doc/comments.js +++ b/tests/doc/comments.js @@ -174,17 +174,19 @@ key2: value 2 k3: v3 #c5\n` const doc = YAML.parseDocument(src) - expect(doc.contents).toMatchObject({ - items: [ - { - commentBefore: 'c0\nc1', - items: [ - {}, - { commentBefore: 'c2', value: { comment: 'c3' } }, - { commentBefore: 'c4' } - ] - } - ], + expect(doc).toMatchObject({ + contents: { + items: [ + { + commentBefore: 'c0\nc1', + items: [ + {}, + { commentBefore: 'c2', value: { comment: 'c3' } }, + { commentBefore: 'c4' } + ] + } + ] + }, comment: 'c5' }) expect(String(doc)).toBe(`#c0 @@ -211,25 +213,28 @@ k2: - v3 #c4 #c5\n` const doc = YAML.parseDocument(src) - expect(doc.contents).toMatchObject({ - items: [ - { - key: { commentBefore: 'c0', value: 'k1' }, - value: { - commentBefore: 'c1', - items: [{ value: 'v1' }, { commentBefore: 'c2', value: 'v2' }], - comment: 'c3' + expect(doc).toMatchObject({ + contents: { + items: [ + { + key: { commentBefore: 'c0', value: 'k1' }, + value: { + commentBefore: 'c1', + items: [{ value: 'v1' }, { commentBefore: 'c2', value: 'v2' }], + comment: 'c3' + } + }, + { + key: { value: 'k2' }, + value: { items: [{ value: 'v3', comment: 'c4' }] } } - }, - { - key: { value: 'k2' }, - value: { items: [{ value: 'v3', comment: 'c4' }] } - } - ], + ] + }, comment: 'c5' }) expect(String(doc)).toBe(`#c0 -k1: #c1 +k1: + #c1 - v1 #c2 - v2 @@ -635,14 +640,15 @@ map: describe('eemeli/yaml#18', () => { test('reported', () => { const src = `test1: - foo: #123 + foo: + #123 bar: 1\n` const doc = YAML.parseDocument(src) expect(String(doc)).toBe(src) }) test('minimal', () => { - const src = `foo: #123\n bar: baz\n` + const src = `foo:\n #123\n bar: baz\n` const doc = YAML.parseDocument(src) expect(String(doc)).toBe(src) }) @@ -714,7 +720,7 @@ c: cc\n` }) }) -describe('collection end comments', () => { +describe.skip('collection end comments', () => { test('seq in seq', () => { const src = `#0 - - a From bb211fc102f59b749ae6599c6efb93b30d7bf511 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 24 Jan 2021 14:32:14 +0200 Subject: [PATCH 71/89] ci: Tweak GitHub Action workflow, for now skipping browser & dist tests --- .github/workflows/browsers.yml | 2 +- .github/workflows/nodejs.yml | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/browsers.yml b/.github/workflows/browsers.yml index 72f87908..8b8cd7a9 100644 --- a/.github/workflows/browsers.yml +++ b/.github/workflows/browsers.yml @@ -1,7 +1,7 @@ name: Browsers on: - - push + # - push - workflow_dispatch jobs: diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 4ad13f7f..1f842aa5 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -1,9 +1,11 @@ name: Node.js on: - - pull_request - - push - - workflow_dispatch + pull_request: + branches: [master] + push: + branches: [master] + workflow_dispatch: jobs: test: @@ -23,7 +25,7 @@ jobs: node-version: ${{ matrix.node-version }} - run: npm ci - run: npm test - - run: npm run test:dist + # - run: npm run test:dist lint: runs-on: ubuntu-latest From 88b6cbdc53e31b69cd83ae476455dd2daa5cc014 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 30 Jan 2021 06:36:09 +0200 Subject: [PATCH 72/89] Rename files in src/parse/, mark class private members as such --- src/compose/compose-collection.ts | 6 +- src/compose/compose-doc.ts | 4 +- src/compose/compose-node.ts | 2 +- src/compose/compose-scalar.ts | 2 +- src/compose/compose-stream.ts | 4 +- src/compose/resolve-block-map.ts | 2 +- src/compose/resolve-block-scalar.ts | 2 +- src/compose/resolve-block-seq.ts | 2 +- src/compose/resolve-end.ts | 2 +- src/compose/resolve-flow-collection.ts | 2 +- src/compose/resolve-flow-scalar.ts | 2 +- src/compose/resolve-props.ts | 2 +- src/compose/util-contains-newline.ts | 2 +- src/constants.d.ts | 8 +- src/parse/{parser.ts => cst-parser.ts} | 158 +++++-------------- src/parse/{parse-stream.ts => cst-stream.ts} | 8 +- src/parse/lexer.ts | 72 ++++----- src/parse/test.ts | 8 +- src/parse/{token-type.ts => tokens.ts} | 133 ++++++++++++++-- 19 files changed, 219 insertions(+), 202 deletions(-) rename src/parse/{parser.ts => cst-parser.ts} (89%) rename src/parse/{parse-stream.ts => cst-stream.ts} (83%) rename src/parse/{token-type.ts => tokens.ts} (51%) diff --git a/src/compose/compose-collection.ts b/src/compose/compose-collection.ts index d928a688..86a0031e 100644 --- a/src/compose/compose-collection.ts +++ b/src/compose/compose-collection.ts @@ -1,10 +1,6 @@ import { Node, Scalar, YAMLMap, YAMLSeq } from '../ast/index.js' import type { Document } from '../doc/Document.js' -import type { - BlockMap, - BlockSequence, - FlowCollection -} from '../parse/parser.js' +import type { BlockMap, BlockSequence, FlowCollection } from '../parse/tokens.js' import { resolveBlockMap } from './resolve-block-map.js' import { resolveBlockSeq } from './resolve-block-seq.js' import { resolveFlowCollection } from './resolve-flow-collection.js' diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index 3615ff9d..5632aec5 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -1,7 +1,7 @@ import { Directives } from '../doc/directives.js' import { Document } from '../doc/Document.js' import type { Options } from '../options.js' -import type * as Parser from '../parse/parser.js' +import type * as Tokens from '../parse/tokens.js' import { composeNode } from './compose-node.js' import { resolveEnd } from './resolve-end.js' import { resolveProps } from './resolve-props.js' @@ -9,7 +9,7 @@ import { resolveProps } from './resolve-props.js' export function composeDoc( options: Options | undefined, directives: Directives, - { offset, start, value, end }: Parser.Document, + { offset, start, value, end }: Tokens.Document, onError: (offset: number, message: string, warning?: boolean) => void ) { const doc = new Document(undefined, options) as Document.Parsed diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts index 0acbf62e..e090ea5f 100644 --- a/src/compose/compose-node.ts +++ b/src/compose/compose-node.ts @@ -1,6 +1,6 @@ import { Alias, Node } from '../ast/index.js' import type { Document } from '../doc/Document.js' -import type { FlowScalar, Token } from '../parse/parser.js' +import type { FlowScalar, Token } from '../parse/tokens.js' import { composeCollection } from './compose-collection.js' import { composeScalar } from './compose-scalar.js' import { resolveEnd } from './resolve-end.js' diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index de2e6d36..6599cd96 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -1,7 +1,7 @@ import { Scalar } from '../ast/index.js' import { Document } from '../doc/Document.js' import type { Schema } from '../doc/Schema.js' -import type { BlockScalar, FlowScalar } from '../parse/parser.js' +import type { BlockScalar, FlowScalar } from '../parse/tokens.js' import { resolveBlockScalar } from './resolve-block-scalar.js' import { resolveFlowScalar } from './resolve-flow-scalar.js' diff --git a/src/compose/compose-stream.ts b/src/compose/compose-stream.ts index 9a7e01d5..4d4ae8de 100644 --- a/src/compose/compose-stream.ts +++ b/src/compose/compose-stream.ts @@ -3,7 +3,7 @@ import { Directives } from '../doc/directives.js' import { Document } from '../doc/Document.js' import { YAMLParseError, YAMLWarning } from '../errors.js' import { defaultOptions, Options } from '../options.js' -import { Parser } from '../parse/parser.js' +import { CSTParser } from '../parse/cst-parser.js' import { composeDoc } from './compose-doc.js' import { resolveEnd } from './resolve-end.js' @@ -104,7 +104,7 @@ export function composeStream( warnings = [] } - const parser = new Parser( + const parser = new CSTParser( token => { if (process.env.LOG_STREAM) console.dir(token, { depth: null }) switch (token.type) { diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index 9c53b76c..f5c5e902 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -1,7 +1,7 @@ import { Pair, YAMLMap } from '../ast/index.js' import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' -import type { BlockMap } from '../parse/parser.js' +import type { BlockMap } from '../parse/tokens.js' import { composeNode } from './compose-node.js' import { resolveMergePair } from './resolve-merge-pair.js' import { resolveProps } from './resolve-props.js' diff --git a/src/compose/resolve-block-scalar.ts b/src/compose/resolve-block-scalar.ts index d428c8c6..007fa9e4 100644 --- a/src/compose/resolve-block-scalar.ts +++ b/src/compose/resolve-block-scalar.ts @@ -1,5 +1,5 @@ import { Type } from '../constants.js' -import type { BlockScalar } from '../parse/parser.js' +import type { BlockScalar } from '../parse/tokens.js' export function resolveBlockScalar( scalar: BlockScalar, diff --git a/src/compose/resolve-block-seq.ts b/src/compose/resolve-block-seq.ts index 2d76e52c..11ee0307 100644 --- a/src/compose/resolve-block-seq.ts +++ b/src/compose/resolve-block-seq.ts @@ -1,7 +1,7 @@ import { YAMLSeq } from '../ast/index.js' import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' -import type { BlockSequence } from '../parse/parser.js' +import type { BlockSequence } from '../parse/tokens.js' import { composeNode } from './compose-node.js' import { resolveProps } from './resolve-props.js' diff --git a/src/compose/resolve-end.ts b/src/compose/resolve-end.ts index 3e661f1d..7ded0d05 100644 --- a/src/compose/resolve-end.ts +++ b/src/compose/resolve-end.ts @@ -1,4 +1,4 @@ -import { SourceToken } from '../parse/parser.js' +import type { SourceToken } from '../parse/tokens.js' export function resolveEnd( end: SourceToken[] | undefined, diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index a460c3e6..e4ed6256 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -1,7 +1,7 @@ import { Node, Pair, YAMLMap, YAMLSeq } from '../ast/index.js' import { Type } from '../constants.js' import type { Document } from '../doc/Document.js' -import type { FlowCollection, SourceToken, Token } from '../parse/parser.js' +import type { FlowCollection, SourceToken, Token } from '../parse/tokens.js' import { composeNode } from './compose-node.js' import { resolveEnd } from './resolve-end.js' import { resolveMergePair } from './resolve-merge-pair.js' diff --git a/src/compose/resolve-flow-scalar.ts b/src/compose/resolve-flow-scalar.ts index 905ad5f1..5a5f05b3 100644 --- a/src/compose/resolve-flow-scalar.ts +++ b/src/compose/resolve-flow-scalar.ts @@ -1,5 +1,5 @@ import { Type } from '../constants.js' -import type { FlowScalar } from '../parse/parser.js' +import type { FlowScalar } from '../parse/tokens.js' import { resolveEnd } from './resolve-end.js' export function resolveFlowScalar( diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts index 2bbd2a87..c2f145ed 100644 --- a/src/compose/resolve-props.ts +++ b/src/compose/resolve-props.ts @@ -1,5 +1,5 @@ import type { Document } from '../doc/Document.js' -import { SourceToken } from '../parse/parser.js' +import type { SourceToken } from '../parse/tokens.js' export function resolveProps( doc: Document.Parsed, diff --git a/src/compose/util-contains-newline.ts b/src/compose/util-contains-newline.ts index 10a08c07..cd0c26ad 100644 --- a/src/compose/util-contains-newline.ts +++ b/src/compose/util-contains-newline.ts @@ -1,4 +1,4 @@ -import type { Token } from '../parse/parser.js' +import type { Token } from '../parse/tokens.js' export function containsNewline(key: Token | null | undefined) { if (!key) return null diff --git a/src/constants.d.ts b/src/constants.d.ts index 3f2977ab..7f26d51a 100644 --- a/src/constants.d.ts +++ b/src/constants.d.ts @@ -28,9 +28,9 @@ export enum Type { SEQ_ITEM = 'SEQ_ITEM' } -export const defaultTagPrefix : 'tag:yaml.org,2002:' -export const defaultTags : { - MAP: 'tag:yaml.org,2002:map', - SEQ: 'tag:yaml.org,2002:seq', +export const defaultTagPrefix: 'tag:yaml.org,2002:' +export const defaultTags: { + MAP: 'tag:yaml.org,2002:map' + SEQ: 'tag:yaml.org,2002:seq' STR: 'tag:yaml.org,2002:str' } diff --git a/src/parse/parser.ts b/src/parse/cst-parser.ts similarity index 89% rename from src/parse/parser.ts rename to src/parse/cst-parser.ts index 0e718718..72169155 100644 --- a/src/parse/parser.ts +++ b/src/parse/cst-parser.ts @@ -1,97 +1,18 @@ import { Lexer } from './lexer.js' -import { SourceTokenType, prettyToken, tokenType } from './token-type.js' - -export interface SourceToken { - type: Exclude - indent: number - source: string -} - -export interface ErrorToken { - type: 'error' - offset?: number - source: string - message: string -} - -export interface Directive { - type: 'directive' - source: string -} - -export interface Document { - type: 'document' - offset: number - start: SourceToken[] - value?: Token - end?: SourceToken[] -} - -export interface DocumentEnd { - type: 'doc-end' - offset: number - source: string - end?: SourceToken[] -} - -export interface FlowScalar { - type: 'alias' | 'scalar' | 'single-quoted-scalar' | 'double-quoted-scalar' - offset: number - indent: number - source: string - end?: SourceToken[] -} - -export interface BlockScalar { - type: 'block-scalar' - offset: number - indent: number - props: Token[] - source?: string -} - -export interface BlockMap { - type: 'block-map' - offset: number - indent: number - items: Array< - | { start: SourceToken[]; key?: never; sep?: never; value?: never } - | { - start: SourceToken[] - key: Token | null - sep: SourceToken[] - value?: Token - } - > -} - -export interface BlockSequence { - type: 'block-seq' - offset: number - indent: number - items: Array<{ start: SourceToken[]; value?: Token; sep?: never }> -} - -export interface FlowCollection { - type: 'flow-collection' - offset: number - indent: number - start: SourceToken - items: Array - end: SourceToken[] -} - -export type Token = - | SourceToken - | ErrorToken - | Directive - | Document - | DocumentEnd - | FlowScalar - | BlockScalar - | BlockMap - | BlockSequence - | FlowCollection +import { + SourceToken, + SourceTokenType, + Token, + FlowScalar, + FlowCollection, + Document, + BlockMap, + BlockScalar, + BlockSequence, + DocumentEnd, + prettyToken, + tokenType +} from './tokens.js' function includesToken(list: SourceToken[], type: SourceToken['type']) { for (let i = 0; i < list.length; ++i) if (list[i].type === type) return true @@ -192,34 +113,35 @@ function getFirstKeyStartProps(prev: SourceToken[]) { } /** A YAML concrete syntax tree parser */ -export class Parser { - push: (token: Token) => void - onNewLine?: (offset: number) => void +export class CSTParser { + private push: (token: Token) => void + private onNewLine?: (offset: number) => void - lexer = new Lexer(ts => this.token(ts)) + private lexer = new Lexer(ts => this.token(ts)) /** If true, space and sequence indicators count as indentation */ - atNewLine = true + private atNewLine = true /** If true, next token is a scalar value */ - atScalar = false + private atScalar = false /** Current indentation level */ - indent = 0 + private indent = 0 + /** Current offset since the start of parsing */ offset = 0 /** On the same line with a block map key */ - onKeyLine = false + private onKeyLine = false /** Top indicates the node that's currently being built */ stack: Token[] = [] /** The source of the current token, set in parse() */ - source = '' + private source = '' /** The type of the current token, set in parse() */ - type = '' as SourceTokenType + private type = '' as SourceTokenType /** * @param push - Called separately with each parsed token @@ -295,7 +217,7 @@ export class Parser { } } - get sourceToken() { + private get sourceToken() { return { type: this.type, indent: this.indent, @@ -303,7 +225,7 @@ export class Parser { } as SourceToken } - step() { + private step() { const top = this.peek(1) if (this.type === 'doc-end' && (!top || top.type !== 'doc-end')) { while (this.stack.length > 0) this.pop() @@ -337,11 +259,11 @@ export class Parser { this.pop() // error } - peek(n: number) { + private peek(n: number) { return this.stack[this.stack.length - n] } - pop(error?: Token) { + private pop(error?: Token) { const token = error || this.stack.pop() if (!token) { const message = 'Tried to pop an empty stack' @@ -415,7 +337,7 @@ export class Parser { } } - stream() { + private stream() { switch (this.type) { case 'directive-line': this.push({ type: 'directive', source: this.source }) @@ -446,7 +368,7 @@ export class Parser { }) } - document(doc: Document) { + private document(doc: Document) { if (doc.value) return this.lineEnd(doc) switch (this.type) { case 'doc-start': { @@ -476,7 +398,7 @@ export class Parser { } } - scalar(scalar: FlowScalar) { + private scalar(scalar: FlowScalar) { if (this.type === 'map-value-ind') { const prev = getPrevProps(this.peek(2)) const start = getFirstKeyStartProps(prev) @@ -499,7 +421,7 @@ export class Parser { } else this.lineEnd(scalar) } - blockScalar(scalar: BlockScalar) { + private blockScalar(scalar: BlockScalar) { switch (this.type) { case 'space': case 'comment': @@ -526,7 +448,7 @@ export class Parser { } } - blockMap(map: BlockMap) { + private blockMap(map: BlockMap) { const it = map.items[map.items.length - 1] // it.sep is true-ish if pair already has key or : separator switch (this.type) { @@ -641,7 +563,7 @@ export class Parser { this.step() } - blockSequence(seq: BlockSequence) { + private blockSequence(seq: BlockSequence) { const it = seq.items[seq.items.length - 1] switch (this.type) { case 'newline': @@ -680,7 +602,7 @@ export class Parser { this.step() } - flowCollection(fc: FlowCollection) { + private flowCollection(fc: FlowCollection) { if (this.type === 'flow-error-end') { let top do { @@ -748,7 +670,7 @@ export class Parser { } } - flowScalar( + private flowScalar( type: 'alias' | 'scalar' | 'single-quoted-scalar' | 'double-quoted-scalar' ) { if (this.onNewLine) { @@ -766,7 +688,7 @@ export class Parser { } as FlowScalar } - startBlockValue(parent: Token) { + private startBlockValue(parent: Token) { switch (this.type) { case 'alias': case 'scalar': @@ -820,7 +742,7 @@ export class Parser { return null } - documentEnd(docEnd: DocumentEnd) { + private documentEnd(docEnd: DocumentEnd) { if (this.type !== 'doc-mode') { if (docEnd.end) docEnd.end.push(this.sourceToken) else docEnd.end = [this.sourceToken] @@ -828,7 +750,7 @@ export class Parser { } } - lineEnd(token: Document | FlowCollection | FlowScalar) { + private lineEnd(token: Document | FlowCollection | FlowScalar) { switch (this.type) { case 'comma': case 'doc-start': diff --git a/src/parse/parse-stream.ts b/src/parse/cst-stream.ts similarity index 83% rename from src/parse/parse-stream.ts rename to src/parse/cst-stream.ts index 2e634fa5..1a23e43b 100644 --- a/src/parse/parse-stream.ts +++ b/src/parse/cst-stream.ts @@ -1,15 +1,15 @@ import { Transform, TransformOptions } from 'stream' import { StringDecoder } from 'string_decoder' -import { Parser } from './parser.js' +import { CSTParser } from './cst-parser.js' export type ParseStreamOptions = Omit< TransformOptions, 'decodeStrings' | 'emitClose' | 'objectMode' > -export class ParseStream extends Transform { +export class CSTStream extends Transform { decoder: StringDecoder - parser: Parser + parser: CSTParser constructor(options: ParseStreamOptions = {}) { super({ @@ -19,7 +19,7 @@ export class ParseStream extends Transform { objectMode: true }) this.decoder = new StringDecoder(options.defaultEncoding || 'utf8') - this.parser = new Parser(token => this.push(token)) + this.parser = new CSTParser(token => this.push(token)) } _flush(done: (error?: Error) => void) { diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index cc3344ce..f9bda91b 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -66,7 +66,7 @@ plain-scalar(is-flow, min) [else] -> plain-scalar(min) */ -import { BOM, DOCUMENT, FLOW_END, SCALAR } from './token-type.js' +import { BOM, DOCUMENT, FLOW_END, SCALAR } from './tokens.js' type State = | 'stream' @@ -97,54 +97,54 @@ const isNotIdentifierChar = (ch: string) => !ch || invalidIdentifierChars.includes(ch) export class Lexer { - push: (token: string) => void + private push: (token: string) => void /** * Flag indicating whether the end of the current buffer marks the end of * all input */ - atEnd = false + private atEnd = false /** * Explicit indent set in block scalar header, as an offset from the current * minimum indent, so e.g. set to 1 from a header `|2+`. Set to -1 if not * explicitly set. */ - blockScalarIndent = -1 + private blockScalarIndent = -1 /** * Block scalars that include a + (keep) chomping indicator in their header * include trailing empty lines, which are otherwise excluded from the * scalar's contents. */ - blockScalarKeep = false + private blockScalarKeep = false /** Current input */ - buffer = '' + private buffer = '' /** * Flag noting whether the map value indicator : can immediately follow this * node within a flow context. */ - flowKey = false + private flowKey = false /** Count of surrounding flow collection levels. */ - flowLevel = 0 + private flowLevel = 0 /** * Minimum level of indentation required for next lines to be parsed as a * part of the current scalar value. */ - indentNext = 0 + private indentNext = 0 /** Indentation level of the current line. */ - indentValue = 0 // actual indent level of current line + private indentValue = 0 /** Stores the state of the lexer if reaching the end of incpomplete input */ - next: State | null = null + private next: State | null = null /** A pointer to `buffer`; the current position of the lexer. */ - pos = 0 + private pos = 0 /** * Define/initialise a YAML lexer. `push` will be called separately with each @@ -170,7 +170,7 @@ export class Lexer { while (next && (incomplete || this.hasChars(1))) next = this.parseNext(next) } - atLineEnd() { + private atLineEnd() { let i = this.pos let ch = this.buffer[i] while (ch === ' ' || ch === '\t') ch = this.buffer[++i] @@ -179,11 +179,11 @@ export class Lexer { return false } - charAt(n: number) { + private charAt(n: number) { return this.buffer[this.pos + n] } - continueScalar(offset: number) { + private continueScalar(offset: number) { let ch = this.buffer[offset] if (this.indentNext > 0) { let indent = 0 @@ -200,29 +200,29 @@ export class Lexer { return offset } - getLine(): string | null { + private getLine(): string | null { let end = this.buffer.indexOf('\n', this.pos) if (end === -1) return this.atEnd ? this.buffer.substring(this.pos) : null if (this.buffer[end - 1] === '\r') end -= 1 return this.buffer.substring(this.pos, end) } - hasChars(n: number) { + private hasChars(n: number) { return this.pos + n <= this.buffer.length } - setNext(state: State) { + private setNext(state: State) { this.buffer = this.buffer.substring(this.pos) this.pos = 0 this.next = state return null } - peek(n: number) { + private peek(n: number) { return this.buffer.substr(this.pos, n) } - parseNext(next: State) { + private parseNext(next: State) { switch (next) { case 'stream': return this.parseStream() @@ -243,7 +243,7 @@ export class Lexer { } } - parseStream() { + private parseStream() { let line = this.getLine() if (line === null) return this.setNext('stream') if (line[0] === BOM) { @@ -277,7 +277,7 @@ export class Lexer { return this.parseLineStart() } - parseLineStart() { + private parseLineStart() { const ch = this.charAt(0) if (ch === '-' || ch === '.') { if (!this.atEnd && !this.hasChars(4)) return this.setNext('line-start') @@ -298,7 +298,7 @@ export class Lexer { return this.parseBlockStart() } - parseBlockStart(): 'doc' | null { + private parseBlockStart(): 'doc' | null { const [ch0, ch1] = this.peek(2) if (!ch1 && !this.atEnd) return this.setNext('block-start') if ((ch0 === '-' || ch0 === '?' || ch0 === ':') && isEmpty(ch1)) { @@ -310,7 +310,7 @@ export class Lexer { return 'doc' } - parseDocument() { + private parseDocument() { this.pushSpaces(true) const line = this.getLine() if (line === null) return this.setNext('doc') @@ -351,7 +351,7 @@ export class Lexer { } } - parseFlowCollection() { + private parseFlowCollection() { let nl: number, sp: number let indent = -1 do { @@ -424,7 +424,7 @@ export class Lexer { } } - parseQuotedScalar() { + private parseQuotedScalar() { const quote = this.charAt(0) let end = this.buffer.indexOf(quote, this.pos + 1) if (quote === "'") { @@ -459,7 +459,7 @@ export class Lexer { return this.flowLevel ? 'flow' : 'doc' } - parseBlockScalarHeader() { + private parseBlockScalarHeader() { this.blockScalarIndent = -1 this.blockScalarKeep = false let i = this.pos @@ -472,7 +472,7 @@ export class Lexer { return this.pushUntil(ch => isEmpty(ch) || ch === '#') } - parseBlockScalar() { + private parseBlockScalar() { let nl = this.pos - 1 let indent = 0 let ch: string @@ -517,7 +517,7 @@ export class Lexer { return this.parseLineStart() } - parsePlainScalar() { + private parsePlainScalar() { const inFlow = this.flowLevel > 0 let end = this.pos - 1 let i = this.pos - 1 @@ -553,7 +553,7 @@ export class Lexer { return inFlow ? 'flow' : 'doc' } - pushCount(n: number) { + private pushCount(n: number) { if (n > 0) { this.push(this.buffer.substr(this.pos, n)) this.pos += n @@ -562,7 +562,7 @@ export class Lexer { return 0 } - pushToIndex(i: number, allowEmpty: boolean) { + private pushToIndex(i: number, allowEmpty: boolean) { const s = this.buffer.slice(this.pos, i) if (s) { this.push(s) @@ -572,7 +572,7 @@ export class Lexer { return 0 } - pushIndicators(): number { + private pushIndicators(): number { switch (this.charAt(0)) { case '!': if (this.charAt(1) === '<') @@ -601,21 +601,21 @@ export class Lexer { return 0 } - pushVerbatimTag() { + private pushVerbatimTag() { let i = this.pos + 2 let ch = this.buffer[i] while (!isEmpty(ch) && ch !== '>') ch = this.buffer[++i] return this.pushToIndex(ch === '>' ? i + 1 : i, false) } - pushNewline() { + private pushNewline() { const ch = this.buffer[this.pos] if (ch === '\n') return this.pushCount(1) else if (ch === '\r' && this.charAt(1) === '\n') return this.pushCount(2) else return 0 } - pushSpaces(allowTabs: boolean) { + private pushSpaces(allowTabs: boolean) { let i = this.pos - 1 let ch: string do { @@ -629,7 +629,7 @@ export class Lexer { return n } - pushUntil(test: (ch: string) => boolean) { + private pushUntil(test: (ch: string) => boolean) { let i = this.pos let ch = this.buffer[i] while (!test(ch)) ch = this.buffer[++i] diff --git a/src/parse/test.ts b/src/parse/test.ts index 62034675..94f899e4 100644 --- a/src/parse/test.ts +++ b/src/parse/test.ts @@ -1,15 +1,15 @@ -import { ParseStream } from './parse-stream.js' -import { Parser } from './parser.js' +import { CSTStream } from './cst-stream.js' +import { CSTParser } from './cst-parser.js' export function stream(source: string) { - const ps = new ParseStream().on('data', d => console.dir(d, { depth: null })) + const ps = new CSTStream().on('data', d => console.dir(d, { depth: null })) ps.write(source) ps.end() } export function test(source: string) { const lines: number[] = [] - const parser = new Parser( + const parser = new CSTParser( t => console.dir(t, { depth: null }), n => lines.push(n) ) diff --git a/src/parse/token-type.ts b/src/parse/tokens.ts similarity index 51% rename from src/parse/token-type.ts rename to src/parse/tokens.ts index 1f5d3c97..fe088cd1 100644 --- a/src/parse/token-type.ts +++ b/src/parse/tokens.ts @@ -1,15 +1,3 @@ -/** The byte order mark */ -export const BOM = '\u{FEFF}' - -/** Start of doc-mode */ -export const DOCUMENT = '\x02' // C0: Start of Text - -/** Unexpected end of flow-mode */ -export const FLOW_END = '\x18' // C0: Cancel - -/** Next token is a scalar value */ -export const SCALAR = '\x1f' // C0: Unit Separator - export type SourceTokenType = | 'byte-order-mark' | 'doc-mode' @@ -36,12 +24,123 @@ export type SourceTokenType = | 'double-quoted-scalar' | 'block-scalar-header' +export interface SourceToken { + type: Exclude + indent: number + source: string +} + +export interface ErrorToken { + type: 'error' + offset?: number + source: string + message: string +} + +export interface Directive { + type: 'directive' + source: string +} + +export interface Document { + type: 'document' + offset: number + start: SourceToken[] + value?: Token + end?: SourceToken[] +} + +export interface DocumentEnd { + type: 'doc-end' + offset: number + source: string + end?: SourceToken[] +} + +export interface FlowScalar { + type: 'alias' | 'scalar' | 'single-quoted-scalar' | 'double-quoted-scalar' + offset: number + indent: number + source: string + end?: SourceToken[] +} + +export interface BlockScalar { + type: 'block-scalar' + offset: number + indent: number + props: Token[] + source?: string +} + +export interface BlockMap { + type: 'block-map' + offset: number + indent: number + items: Array< + | { start: SourceToken[]; key?: never; sep?: never; value?: never } + | { + start: SourceToken[] + key: Token | null + sep: SourceToken[] + value?: Token + } + > +} + +export interface BlockSequence { + type: 'block-seq' + offset: number + indent: number + items: Array<{ start: SourceToken[]; value?: Token; sep?: never }> +} + +export interface FlowCollection { + type: 'flow-collection' + offset: number + indent: number + start: SourceToken + items: Array + end: SourceToken[] +} + +export type Token = + | SourceToken + | ErrorToken + | Directive + | Document + | DocumentEnd + | FlowScalar + | BlockScalar + | BlockMap + | BlockSequence + | FlowCollection + +/** The byte order mark */ +export const BOM = '\u{FEFF}' + +/** Start of doc-mode */ +export const DOCUMENT = '\x02' // C0: Start of Text + +/** Unexpected end of flow-mode */ +export const FLOW_END = '\x18' // C0: Cancel + +/** Next token is a scalar value */ +export const SCALAR = '\x1f' // C0: Unit Separator + export function prettyToken(token: string) { - if (token === BOM) return '' - if (token === DOCUMENT) return '' - if (token === FLOW_END) return '' - if (token === SCALAR) return '' - return JSON.stringify(token) + switch (token) { + case BOM: + return '' + case DOCUMENT: + return '' + case FLOW_END: + return '' + case SCALAR: + return '' + default: + return JSON.stringify(token) + } } export function tokenType(source: string): SourceTokenType | null { From 2bfb11cd789f72bf8b033f761c510f384ad56662 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 31 Jan 2021 23:54:43 +0200 Subject: [PATCH 73/89] Refactor visit() as TypeScript --- rollup.dev-config.js | 3 +- src/visit.d.ts | 60 ------------------- src/visit.js | 79 ------------------------- src/visit.ts | 138 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 141 deletions(-) delete mode 100644 src/visit.d.ts delete mode 100644 src/visit.js create mode 100644 src/visit.ts diff --git a/rollup.dev-config.js b/rollup.dev-config.js index f244b14a..651229cf 100644 --- a/rollup.dev-config.js +++ b/rollup.dev-config.js @@ -6,8 +6,7 @@ export default { 'src/ast/index.js', 'src/doc/Document.js', 'src/errors.js', - 'src/options.js', - 'src/visit.js' + 'src/options.js' ], external: [resolve('src/doc/directives.js')], output: { dir: 'lib', format: 'cjs', esModule: false, preserveModules: true }, diff --git a/src/visit.d.ts b/src/visit.d.ts deleted file mode 100644 index 49b6ce15..00000000 --- a/src/visit.d.ts +++ /dev/null @@ -1,60 +0,0 @@ -import type { Alias, Pair, Scalar, YAMLMap, YAMLSeq } from './ast/index.js' - -/** - * Apply a visitor to an AST node or document. - * - * Walks through the tree (depth-first) starting from `node`, calling a - * `visitor` function with three arguments: - * - `key`: For sequence values and map `Pair`, the node's index in the - * collection. Within a `Pair`, `'key'` or `'value'`, correspondingly. - * `null` for the root node. - * - `node`: The current node. - * - `path`: The ancestry of the current node. - * - * The return value of the visitor may be used to control the traversal: - * - `undefined` (default): Do nothing and continue - * - `visit.SKIP`: Do not visit the children of this node, continue with next - * sibling - * - `visit.BREAK`: Terminate traversal completely - * - `visit.REMOVE`: Remove the current node, then continue with the next one - * - `Node`: Replace the current node, then continue by visiting it - * - `number`: While iterating the items of a sequence or map, set the index - * of the next step. This is useful especially if the index of the current - * node has changed. - * - * If `visitor` is a single function, it will be called with all values - * encountered in the tree, including e.g. `null` values. Alternatively, - * separate visitor functions may be defined for each `Map`, `Pair`, `Seq`, - * `Alias` and `Scalar` node. - */ -export declare const visit: visit - -export type visitor = ( - key: number | 'key' | 'value' | null, - node: T, - path: Node[] -) => void | symbol | number | Node - -export interface visit { - ( - node: Node | Document, - visitor: - | visitor - | { - Alias?: visitor - Map?: visitor - Pair?: visitor - Scalar?: visitor - Seq?: visitor - } - ): void - - /** Terminate visit traversal completely */ - BREAK: symbol - - /** Remove the current node */ - REMOVE: symbol - - /** Do not visit the children of the current node */ - SKIP: symbol -} diff --git a/src/visit.js b/src/visit.js deleted file mode 100644 index b190c7a1..00000000 --- a/src/visit.js +++ /dev/null @@ -1,79 +0,0 @@ -import { Alias } from './ast/Alias.js' -import { Node } from './ast/Node.js' -import { Pair } from './ast/Pair.js' -import { Scalar } from './ast/Scalar.js' -import { YAMLMap } from './ast/YAMLMap.js' -import { YAMLSeq } from './ast/YAMLSeq.js' -import { Document } from './doc/Document.js' - -const BREAK = Symbol('break visit') -const SKIP = Symbol('skip children') -const REMOVE = Symbol('remove node') - -function _visit(key, node, visitor, path) { - let ctrl = undefined - if (typeof visitor === 'function') ctrl = visitor(key, node, path) - else if (node instanceof YAMLMap) { - if (visitor.Map) ctrl = visitor.Map(key, node, path) - } else if (node instanceof YAMLSeq) { - if (visitor.Seq) ctrl = visitor.Seq(key, node, path) - } else if (node instanceof Pair) { - if (visitor.Pair) ctrl = visitor.Pair(key, node, path) - } else if (node instanceof Scalar) { - if (visitor.Scalar) ctrl = visitor.Scalar(key, node, path) - } else if (node instanceof Alias) { - if (visitor.Scalar) ctrl = visitor.Alias(key, node, path) - } - - if (ctrl instanceof Node) { - const parent = path[path.length - 1] - if (parent instanceof YAMLMap || parent instanceof YAMLSeq) { - parent.items[key] = ctrl - } else if (parent instanceof Pair) { - if (key === 'key') parent.key = ctrl - else parent.value = ctrl - } else if (parent instanceof Document) { - parent.contents = ctrl - } else { - const pt = parent && parent.type - throw new Error(`Cannot replace node with ${pt} parent`) - } - return _visit(key, ctrl, visitor, path) - } - - if (typeof ctrl !== 'symbol') { - if (node instanceof YAMLMap || node instanceof YAMLSeq) { - path = Object.freeze(path.concat(node)) - for (let i = 0; i < node.items.length; ++i) { - const ci = _visit(i, node.items[i], visitor, path) - if (typeof ci === 'number') i = ci - 1 - else if (ci === BREAK) return BREAK - else if (ci === REMOVE) { - node.items.splice(i, 1) - i -= 1 - } - } - } else if (node instanceof Pair) { - path = Object.freeze(path.concat(node)) - const ck = _visit('key', node.key, visitor, path) - if (ck === BREAK) return BREAK - else if (ck === REMOVE) node.key = null - const cv = _visit('value', node.value, visitor, path) - if (cv === BREAK) return BREAK - else if (cv === REMOVE) node.value = null - } - } - - return ctrl -} - -export function visit(node, visitor) { - if (node instanceof Document) { - const cd = _visit(null, node.contents, visitor, Object.freeze([node])) - if (cd === REMOVE) node.contents = null - } else _visit(null, node, visitor, Object.freeze([])) -} - -visit.BREAK = BREAK -visit.SKIP = SKIP -visit.REMOVE = REMOVE diff --git a/src/visit.ts b/src/visit.ts new file mode 100644 index 00000000..0ac4701b --- /dev/null +++ b/src/visit.ts @@ -0,0 +1,138 @@ +import { Alias, Node, Pair, Scalar, YAMLMap, YAMLSeq } from './ast/index.js' +import { Document } from './doc/Document.js' + +const BREAK = Symbol('break visit') +const SKIP = Symbol('skip children') +const REMOVE = Symbol('remove node') + +export type visitorFn = ( + key: number | 'key' | 'value' | null, + node: T, + path: readonly Node[] +) => void | symbol | number | Node + +export type visitor = + | visitorFn + | { + Alias?: visitorFn + Map?: visitorFn + Pair?: visitorFn + Scalar?: visitorFn + Seq?: visitorFn + } + +/** + * Apply a visitor to an AST node or document. + * + * Walks through the tree (depth-first) starting from `node`, calling a + * `visitor` function with three arguments: + * - `key`: For sequence values and map `Pair`, the node's index in the + * collection. Within a `Pair`, `'key'` or `'value'`, correspondingly. + * `null` for the root node. + * - `node`: The current node. + * - `path`: The ancestry of the current node. + * + * The return value of the visitor may be used to control the traversal: + * - `undefined` (default): Do nothing and continue + * - `visit.SKIP`: Do not visit the children of this node, continue with next + * sibling + * - `visit.BREAK`: Terminate traversal completely + * - `visit.REMOVE`: Remove the current node, then continue with the next one + * - `Node`: Replace the current node, then continue by visiting it + * - `number`: While iterating the items of a sequence or map, set the index + * of the next step. This is useful especially if the index of the current + * node has changed. + * + * If `visitor` is a single function, it will be called with all values + * encountered in the tree, including e.g. `null` values. Alternatively, + * separate visitor functions may be defined for each `Map`, `Pair`, `Seq`, + * `Alias` and `Scalar` node. + */ +export function visit( + node: Node | Document, + visitor: + | visitorFn + | { + Alias?: visitorFn + Map?: visitorFn + Pair?: visitorFn + Scalar?: visitorFn + Seq?: visitorFn + } +) { + if (node instanceof Document) { + const cd = _visit(null, node.contents, visitor, Object.freeze([node])) + if (cd === REMOVE) node.contents = null + } else _visit(null, node, visitor, Object.freeze([])) +} + +/** Terminate visit traversal completely */ +visit.BREAK = BREAK + +/** Do not visit the children of the current node */ +visit.SKIP = SKIP + +/** Remove the current node */ +visit.REMOVE = REMOVE + +function _visit( + key: number | 'key' | 'value' | null, + node: Node, + visitor: visitor, + path: readonly Node[] +): void | symbol | number | Node { + let ctrl = undefined + if (typeof visitor === 'function') ctrl = visitor(key, node, path) + else if (node instanceof YAMLMap) { + if (visitor.Map) ctrl = visitor.Map(key, node, path) + } else if (node instanceof YAMLSeq) { + if (visitor.Seq) ctrl = visitor.Seq(key, node, path) + } else if (node instanceof Pair) { + if (visitor.Pair) ctrl = visitor.Pair(key, node, path) + } else if (node instanceof Scalar) { + if (visitor.Scalar) ctrl = visitor.Scalar(key, node, path) + } else if (node instanceof Alias) { + if (visitor.Alias) ctrl = visitor.Alias(key, node, path) + } + + if (ctrl instanceof Node) { + const parent = path[path.length - 1] + if (parent instanceof YAMLMap || parent instanceof YAMLSeq) { + parent.items[key as number] = ctrl + } else if (parent instanceof Pair) { + if (key === 'key') parent.key = ctrl + else parent.value = ctrl + } else if (parent instanceof Document) { + parent.contents = ctrl + } else { + const pt = parent && parent.type + throw new Error(`Cannot replace node with ${pt} parent`) + } + return _visit(key, ctrl, visitor, path) + } + + if (typeof ctrl !== 'symbol') { + if (node instanceof YAMLMap || node instanceof YAMLSeq) { + path = Object.freeze(path.concat(node)) + for (let i = 0; i < node.items.length; ++i) { + const ci = _visit(i, node.items[i], visitor, path) + if (typeof ci === 'number') i = ci - 1 + else if (ci === BREAK) return BREAK + else if (ci === REMOVE) { + node.items.splice(i, 1) + i -= 1 + } + } + } else if (node instanceof Pair) { + path = Object.freeze(path.concat(node)) + const ck = _visit('key', node.key, visitor, path) + if (ck === BREAK) return BREAK + else if (ck === REMOVE) node.key = null + const cv = _visit('value', node.value, visitor, path) + if (cv === BREAK) return BREAK + else if (cv === REMOVE) node.value = null + } + } + + return ctrl +} From 87e0e23f4a466f1a6bbc50631fd361e7d12d4146 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 1 Feb 2021 01:48:58 +0200 Subject: [PATCH 74/89] Include source string value for all scalars --- src/ast/index.d.ts | 1 + src/compose/compose-scalar.ts | 9 ++++---- src/tags/core.js | 33 +++++++++------------------ src/tags/yaml-1.1/index.js | 43 +++++++++++++++-------------------- 4 files changed, 35 insertions(+), 51 deletions(-) diff --git a/src/ast/index.d.ts b/src/ast/index.d.ts index f1f3931a..e6868f35 100644 --- a/src/ast/index.d.ts +++ b/src/ast/index.d.ts @@ -46,6 +46,7 @@ export class Scalar extends Node { export namespace Scalar { interface Parsed extends Scalar { range: [number, number] + source: string } type Type = | Type.BLOCK_FOLDED diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index 6599cd96..4e9fa2ee 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -22,22 +22,23 @@ export function composeScalar( ? findScalarTagByName(doc.schema, value, tagName, onError) : findScalarTagByTest(doc.schema, value, token.type === 'scalar') - let scalar: Scalar + let scalar: Scalar.Parsed try { const res = tag ? tag.resolve(value, msg => onError(offset, msg)) : value - scalar = res instanceof Scalar ? res : new Scalar(res) + scalar = (res instanceof Scalar ? res : new Scalar(res)) as Scalar.Parsed } catch (error) { onError(offset, error.message) - scalar = new Scalar(value) + scalar = new Scalar(value) as Scalar.Parsed } scalar.range = [offset, offset + length] + scalar.source = value if (type) scalar.type = type if (tagName) scalar.tag = tagName if (tag?.format) scalar.format = tag.format if (comment) scalar.comment = comment if (anchor) doc.anchors.setAnchor(scalar, anchor) - return scalar as Scalar.Parsed + return scalar } const defaultScalarTag = (schema: Schema) => diff --git a/src/tags/core.js b/src/tags/core.js index 33cb59fb..1b3b84de 100644 --- a/src/tags/core.js +++ b/src/tags/core.js @@ -17,16 +17,6 @@ function intStringify(node, radix, prefix) { return stringifyNumber(node) } -function stringifyBool(node) { - const { value, sourceStr } = node - if (sourceStr) { - const match = boolObj.test.test(sourceStr) - if (match && value === (sourceStr[0] === 't' || sourceStr[0] === 'T')) - return sourceStr - } - return value ? boolOptions.trueStr : boolOptions.falseStr -} - export const nullObj = { identify: value => value == null, createNode: (schema, value, ctx) => @@ -34,13 +24,10 @@ export const nullObj = { default: true, tag: 'tag:yaml.org,2002:null', test: /^(?:~|[Nn]ull|NULL)?$/, - resolve: str => { - const node = new Scalar(null) - if (str) node.sourceStr = str - return node - }, + resolve: () => new Scalar(null), options: nullOptions, - stringify: ({ sourceStr }) => sourceStr ?? nullOptions.nullStr + stringify: ({ source }) => + source && nullObj.test.test(source) ? source : nullOptions.nullStr } export const boolObj = { @@ -48,13 +35,15 @@ export const boolObj = { default: true, tag: 'tag:yaml.org,2002:bool', test: /^(?:[Tt]rue|TRUE|[Ff]alse|FALSE)$/, - resolve: str => { - const node = new Scalar(str[0] === 't' || str[0] === 'T') - node.sourceStr = str - return node - }, + resolve: str => new Scalar(str[0] === 't' || str[0] === 'T'), options: boolOptions, - stringify: stringifyBool + stringify({ source, value }) { + if (source && boolObj.test.test(source)) { + const sv = source[0] === 't' || source[0] === 'T' + if (value === sv) return source + } + return value ? boolOptions.trueStr : boolOptions.falseStr + } } export const octObj = { diff --git a/src/tags/yaml-1.1/index.js b/src/tags/yaml-1.1/index.js index 2781a693..358eb1b7 100644 --- a/src/tags/yaml-1.1/index.js +++ b/src/tags/yaml-1.1/index.js @@ -10,16 +10,23 @@ import { pairs } from './pairs.js' import { set } from './set.js' import { intTime, floatTime, timestamp } from './timestamp.js' -const boolStringify = ({ value, sourceStr }) => { - const boolObj = value ? trueObj : falseObj - if (sourceStr && boolObj.test.test(sourceStr)) return sourceStr - return value ? boolOptions.trueStr : boolOptions.falseStr +const nullObj = { + identify: value => value == null, + createNode: (schema, value, ctx) => + ctx.wrapScalars ? new Scalar(null) : null, + default: true, + tag: 'tag:yaml.org,2002:null', + test: /^(?:~|[Nn]ull|NULL)?$/, + resolve: () => new Scalar(null), + options: nullOptions, + stringify: ({ source }) => + source && nullObj.test.test(source) ? source : nullOptions.nullStr } -const boolResolve = (value, str) => { - const node = new Scalar(value) - node.sourceStr = str - return node +const boolStringify = ({ value, source }) => { + const boolObj = value ? trueObj : falseObj + if (source && boolObj.test.test(source)) return source + return value ? boolOptions.trueStr : boolOptions.falseStr } const trueObj = { @@ -27,7 +34,7 @@ const trueObj = { default: true, tag: 'tag:yaml.org,2002:bool', test: /^(?:Y|y|[Yy]es|YES|[Tt]rue|TRUE|[Oo]n|ON)$/, - resolve: str => boolResolve(true, str), + resolve: () => new Scalar(true), options: boolOptions, stringify: boolStringify } @@ -37,7 +44,7 @@ const falseObj = { default: true, tag: 'tag:yaml.org,2002:bool', test: /^(?:N|n|[Nn]o|NO|[Ff]alse|FALSE|[Oo]ff|OFF)$/i, - resolve: str => boolResolve(false, str), + resolve: () => new Scalar(false), options: boolOptions, stringify: boolStringify } @@ -79,21 +86,7 @@ function intStringify(node, radix, prefix) { export const yaml11 = failsafe.concat( [ - { - identify: value => value == null, - createNode: (schema, value, ctx) => - ctx.wrapScalars ? new Scalar(null) : null, - default: true, - tag: 'tag:yaml.org,2002:null', - test: /^(?:~|[Nn]ull|NULL)?$/, - resolve: str => { - const node = new Scalar(null) - if (str) node.sourceStr = str - return node - }, - options: nullOptions, - stringify: ({ sourceStr }) => sourceStr ?? nullOptions.nullStr - }, + nullObj, trueObj, falseObj, { From db96f631b4d49628f006a5764866ef6ef79472b5 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 1 Feb 2021 01:53:07 +0200 Subject: [PATCH 75/89] Use new parser for test-events; fix bugs as needed --- src/compose/compose-stream.ts | 1 + src/compose/resolve-flow-collection.ts | 2 ++ src/test-events.js | 29 +++++++++----------------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/compose/compose-stream.ts b/src/compose/compose-stream.ts index 4d4ae8de..a52c562d 100644 --- a/src/compose/compose-stream.ts +++ b/src/compose/compose-stream.ts @@ -154,6 +154,7 @@ export function composeStream( const dc = doc.comment doc.comment = dc ? `${dc}\n${end.comment}` : end.comment } + doc.range[1] = end.offset break } default: diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index e4ed6256..2fb9ea63 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -62,6 +62,7 @@ export function resolveFlowCollection( const seq = coll as YAMLSeq if (key) { const map = new YAMLMap(doc.schema) + map.type = Type.FLOW_MAP map.items.push(new Pair(key, value)) seq.items.push(map) } else seq.items.push(value) @@ -135,6 +136,7 @@ export function resolveFlowCollection( if (value) { onError(offset, 'Missing {} around pair used as mapping key') const map = new YAMLMap(doc.schema) + map.type = Type.FLOW_MAP map.items.push(new Pair(key, value)) map.range = [key.range[0], value.range[1]] key = map as YAMLMap.Parsed diff --git a/src/test-events.js b/src/test-events.js index 62fe9f0b..081eec01 100644 --- a/src/test-events.js +++ b/src/test-events.js @@ -1,13 +1,9 @@ -import { parse } from './cst/parse.js' -import { Document } from './doc/Document.js' +import { parseAllDocuments } from './index.js' // test harness for yaml-test-suite event tests export function testEvents(src, options) { - const opt = Object.assign( - { keepCstNodes: true, keepNodeTypes: true, version: '1.2' }, - options - ) - const docs = parse(src).map(cstDoc => new Document(null, opt).parse(cstDoc)) + const opt = Object.assign({ keepNodeTypes: true, version: '1.2' }, options) + const docs = parseAllDocuments(src, opt) const errDoc = docs.find(doc => doc.errors.length > 0) const error = errDoc ? errDoc.errors[0].message : null const events = ['+STR'] @@ -22,17 +18,15 @@ export function testEvents(src, options) { if (e && (e.type === 'DOCUMENT' || e.range.start < rootStart)) throw new Error() let docStart = '+DOC' - const pre = src.slice(0, rootStart) - const explicitDoc = /---\s*$/.test(pre) - if (explicitDoc) docStart += ' ---' - else if (!doc.contents) continue + if (doc.directivesEndMarker) docStart += ' ---' + else if (doc.contents.range[1] === doc.contents.range[0]) continue events.push(docStart) addEvents(events, doc, e, root) if (doc.contents && doc.contents.length > 1) throw new Error() let docEnd = '-DOC' if (rootEnd) { - const post = src.slice(rootEnd) - if (/^\.\.\./.test(post)) docEnd += ' ...' + const post = src.slice(rootStart, rootEnd) + if (/^\.\.\.($|\s)/m.test(post)) docEnd += ' ...' } events.push(docEnd) } @@ -48,7 +42,7 @@ function addEvents(events, doc, e, node) { events.push('=VAL :') return } - if (e && node.cstNode === e) throw new Error() + if (e /*&& node.cstNode === e*/) throw new Error() let props = '' let anchor = doc.anchors.getName(node) if (anchor) { @@ -58,10 +52,7 @@ function addEvents(events, doc, e, node) { } props = ` &${anchor}` } - if (node.cstNode && node.cstNode.tag) { - const { handle, suffix } = node.cstNode.tag - props += handle === '!' && !suffix ? ' ' : ` <${node.tag}>` - } + if (node.tag) props += ` <${node.tag}>` let scalar = null switch (node.type) { case 'ALIAS': @@ -116,7 +107,7 @@ function addEvents(events, doc, e, node) { throw new Error(`Unexpected node type ${node.type}`) } if (scalar) { - const value = node.cstNode.strValue + const value = node.source .replace(/\\/g, '\\\\') .replace(/\0/g, '\\0') .replace(/\x07/g, '\\a') From fec3ca00a326b5155ba22ceba68a4cfa8341a2e8 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 1 Feb 2021 01:54:21 +0200 Subject: [PATCH 76/89] Drop Document#parse(cst) method --- index.d.ts | 2 - src/doc/Document.d.ts | 2 - src/doc/Document.js | 45 ++-------------------- src/doc/parseContents.js | 55 -------------------------- src/doc/parseDirectives.js | 79 -------------------------------------- 5 files changed, 3 insertions(+), 180 deletions(-) delete mode 100644 src/doc/parseContents.js delete mode 100644 src/doc/parseDirectives.js diff --git a/index.d.ts b/index.d.ts index 5058d068..f7e8a41b 100644 --- a/index.d.ts +++ b/index.d.ts @@ -348,8 +348,6 @@ export class Document extends Collection { */ createPair(key: any, value: any, options?: { wrapScalars?: boolean }): Pair - /** Parse a CST into this document */ - parse(cst: CST.Document): this /** * When a document is created with `new YAML.Document()`, the schema object is * not set as it may be influenced by parsed directives; call this with no diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts index 49b97878..864d8051 100644 --- a/src/doc/Document.d.ts +++ b/src/doc/Document.d.ts @@ -88,8 +88,6 @@ export class Document extends Collection { */ createPair(key: any, value: any, options?: { wrapScalars?: boolean }): Pair - /** Parse a CST into this document */ - parse(cst: CST.Document): this /** * When a document is created with `new YAML.Document()`, the schema object is * not set as it may be influenced by parsed directives; call this with no diff --git a/src/doc/Document.js b/src/doc/Document.js index 4d5b04be..e2856329 100644 --- a/src/doc/Document.js +++ b/src/doc/Document.js @@ -8,8 +8,6 @@ import { isEmptyPath, toJS } from '../ast/index.js' -import { Document as CSTDocument } from '../cst/Document.js' -import { YAMLError } from '../errors.js' import { defaultOptions, documentOptions } from '../options.js' import { addComment } from '../stringify/addComment.js' import { stringify } from '../stringify/stringify.js' @@ -19,8 +17,6 @@ import { Schema } from './Schema.js' import { applyReviver } from './applyReviver.js' import { createNode } from './createNode.js' import { Directives } from './directives.js' -import { parseContents } from './parseContents.js' -import { parseDirectives } from './parseDirectives.js' function assertCollection(contents) { if (contents instanceof Collection) return true @@ -52,14 +48,9 @@ export class Document { this.tagPrefixes = [] this.warnings = [] - if (value === undefined) { - // note that this.schema is left as null here - this.contents = null - } else if (value instanceof CSTDocument) { - this.parse(value) - } else { - this.contents = this.createNode(value, { replacer }) - } + // note that this.schema is left as null here + this.contents = + value === undefined ? null : this.createNode(value, { replacer }) } add(value) { @@ -207,36 +198,6 @@ export class Document { this.schema = new Schema(opt) } - parse(node, prevDoc) { - if (this.options.keepCstNodes) this.cstNode = node - if (this.options.keepNodeTypes) this.type = 'DOCUMENT' - const { - directives = [], - contents = [], - directivesEndMarker, - error, - valueRange - } = node - if (error) { - if (!error.source) error.source = this - this.errors.push(error) - } - parseDirectives(this, directives, prevDoc) - if (directivesEndMarker) this.directivesEndMarker = true - this.range = valueRange ? [valueRange.start, valueRange.end] : null - this.setSchema() - this.anchors._cstAliases = [] - parseContents(this, contents) - this.anchors.resolveNodes() - if (this.options.prettyErrors) { - for (const error of this.errors) - if (error instanceof YAMLError) error.makePretty() - for (const warn of this.warnings) - if (warn instanceof YAMLError) warn.makePretty() - } - return this - } - setTagPrefix(handle, prefix) { if (handle[0] !== '!' || handle[handle.length - 1] !== '!') throw new Error('Handle must start and end with !') diff --git a/src/doc/parseContents.js b/src/doc/parseContents.js deleted file mode 100644 index 7c2fe1e2..00000000 --- a/src/doc/parseContents.js +++ /dev/null @@ -1,55 +0,0 @@ -import { Collection } from '../ast/index.js' -import { Type } from '../constants.js' -import { YAMLSyntaxError } from '../errors.js' -import { resolveNode } from '../resolve/resolveNode.js' - -export function parseContents(doc, contents) { - const comments = { before: [], after: [] } - let body = undefined - let spaceBefore = false - for (const node of contents) { - if (node.valueRange) { - if (body !== undefined) { - const msg = - 'Document contains trailing content not separated by a ... or --- line' - doc.errors.push(new YAMLSyntaxError(node, msg)) - break - } - const res = resolveNode(doc, node) - if (spaceBefore) { - res.spaceBefore = true - spaceBefore = false - } - body = res - } else if (node.comment !== null) { - const cc = body === undefined ? comments.before : comments.after - cc.push(node.comment) - } else if (node.type === Type.BLANK_LINE) { - spaceBefore = true - if ( - body === undefined && - comments.before.length > 0 && - !doc.commentBefore - ) { - // space-separated comments at start are parsed as document comments - doc.commentBefore = comments.before.join('\n') - comments.before = [] - } - } - } - - doc.contents = body || null - if (!body) { - doc.comment = comments.before.concat(comments.after).join('\n') || null - } else { - const cb = comments.before.join('\n') - if (cb) { - const cbNode = - body instanceof Collection && body.items[0] ? body.items[0] : body - cbNode.commentBefore = cbNode.commentBefore - ? `${cb}\n${cbNode.commentBefore}` - : cb - } - doc.comment = comments.after.join('\n') || null - } -} diff --git a/src/doc/parseDirectives.js b/src/doc/parseDirectives.js deleted file mode 100644 index 995fb739..00000000 --- a/src/doc/parseDirectives.js +++ /dev/null @@ -1,79 +0,0 @@ -import { YAMLSemanticError, YAMLWarning } from '../errors.js' -import { documentOptions } from '../options.js' - -function resolveTagDirective({ tagPrefixes }, directive) { - const [handle, prefix] = directive.parameters - if (!handle || !prefix) { - const msg = 'Insufficient parameters given for %TAG directive' - throw new YAMLSemanticError(directive, msg) - } - if (tagPrefixes.some(p => p.handle === handle)) { - const msg = - 'The %TAG directive must only be given at most once per handle in the same document.' - throw new YAMLSemanticError(directive, msg) - } - return { handle, prefix } -} - -function resolveYamlDirective(doc, directive) { - let [version] = directive.parameters - if (directive.name === 'YAML:1.0') version = '1.0' - if (!version) { - const msg = 'Insufficient parameters given for %YAML directive' - throw new YAMLSemanticError(directive, msg) - } - if (!documentOptions[version]) { - const v0 = doc.version || doc.options.version - const msg = `Document will be parsed as YAML ${v0} rather than YAML ${version}` - doc.warnings.push(new YAMLWarning(directive, msg)) - } - return version -} - -export function parseDirectives(doc, directives, prevDoc) { - const directiveComments = [] - let hasDirectives = false - for (const directive of directives) { - const { comment, name } = directive - switch (name) { - case 'TAG': - try { - doc.tagPrefixes.push(resolveTagDirective(doc, directive)) - } catch (error) { - doc.errors.push(error) - } - hasDirectives = true - break - case 'YAML': - case 'YAML:1.0': - if (doc.version) { - const msg = - 'The %YAML directive must only be given at most once per document.' - doc.errors.push(new YAMLSemanticError(directive, msg)) - } - try { - doc.version = resolveYamlDirective(doc, directive) - } catch (error) { - doc.errors.push(error) - } - hasDirectives = true - break - default: - if (name) { - const msg = `YAML only supports %TAG and %YAML directives, and not %${name}` - doc.warnings.push(new YAMLWarning(directive, msg)) - } - } - if (comment) directiveComments.push(comment) - } - if ( - prevDoc && - !hasDirectives && - '1.1' === (doc.version || prevDoc.version || doc.options.version) - ) { - const copyTagPrefix = ({ handle, prefix }) => ({ handle, prefix }) - doc.tagPrefixes = prevDoc.tagPrefixes.map(copyTagPrefix) - doc.version = prevDoc.version - } - doc.commentBefore = directiveComments.join('\n') || null -} From 15f88714065b5ff68e0b1ced7ff8c14ca942d8c0 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 1 Feb 2021 02:04:30 +0200 Subject: [PATCH 77/89] Only test for default tag matches when stringifying a string --- src/stringify/stringifyString.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/stringify/stringifyString.js b/src/stringify/stringifyString.js index 8d2ad65f..62b6c183 100644 --- a/src/stringify/stringifyString.js +++ b/src/stringify/stringifyString.js @@ -1,6 +1,5 @@ import { addCommentBefore } from './addComment.js' import { Type } from '../constants.js' -import { resolveScalar } from '../resolve/resolveScalar.js' import { foldFlowLines, FOLD_BLOCK, @@ -259,9 +258,14 @@ function plainString(item, ctx, onComment, onChompKeep) { // booleans get parsed with those types in v1.2 (e.g. '42', 'true' & '0.9e-3'), // and others in v1.1. if (actualString) { - const { tags } = ctx.doc.schema - const resolved = resolveScalar(str, tags).value - if (typeof resolved !== 'string') return doubleQuotedString(value, ctx) + for (const tag of ctx.doc.schema.tags) { + if ( + tag.default && + tag.tag !== 'tag:yaml.org,2002:str' && + tag.test?.test(str) + ) + return doubleQuotedString(value, ctx) + } } const body = implicitKey ? str From dd0198ec1470829640501f688dce144436c3ced4 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 1 Feb 2021 02:05:25 +0200 Subject: [PATCH 78/89] Drop all code in src/resolve/ as unused --- src/resolve/collection-utils.js | 76 ---------- src/resolve/resolveMap.js | 257 -------------------------------- src/resolve/resolveNode.js | 145 ------------------ src/resolve/resolveScalar.js | 13 -- src/resolve/resolveSeq.js | 139 ----------------- src/resolve/resolveTag.js | 95 ------------ src/resolve/resolveTagName.js | 75 ---------- 7 files changed, 800 deletions(-) delete mode 100644 src/resolve/collection-utils.js delete mode 100644 src/resolve/resolveMap.js delete mode 100644 src/resolve/resolveNode.js delete mode 100644 src/resolve/resolveScalar.js delete mode 100644 src/resolve/resolveSeq.js delete mode 100644 src/resolve/resolveTag.js delete mode 100644 src/resolve/resolveTagName.js diff --git a/src/resolve/collection-utils.js b/src/resolve/collection-utils.js deleted file mode 100644 index baf6a1c3..00000000 --- a/src/resolve/collection-utils.js +++ /dev/null @@ -1,76 +0,0 @@ -import { YAMLSemanticError } from '../errors.js' -import { Type } from '../constants.js' - -export function checkFlowCollectionEnd(errors, cst) { - let char, name - switch (cst.type) { - case Type.FLOW_MAP: - char = '}' - name = 'flow map' - break - case Type.FLOW_SEQ: - char = ']' - name = 'flow sequence' - break - default: - errors.push(new YAMLSemanticError(cst, 'Not a flow collection!?')) - return - } - - let lastItem - for (let i = cst.items.length - 1; i >= 0; --i) { - const item = cst.items[i] - if (!item || item.type !== Type.COMMENT) { - lastItem = item - break - } - } - - if (lastItem && lastItem.char !== char) { - const msg = `Expected ${name} to end with ${char}` - let err - if (typeof lastItem.offset === 'number') { - err = new YAMLSemanticError(cst, msg) - err.offset = lastItem.offset + 1 - } else { - err = new YAMLSemanticError(lastItem, msg) - if (lastItem.range && lastItem.range.end) - err.offset = lastItem.range.end - lastItem.range.start - } - errors.push(err) - } -} -export function checkFlowCommentSpace(errors, comment) { - const prev = comment.context.src[comment.range.start - 1] - if (prev !== '\n' && prev !== '\t' && prev !== ' ') { - const msg = - 'Comments must be separated from other tokens by white space characters' - errors.push(new YAMLSemanticError(comment, msg)) - } -} - -export function getLongKeyError(source, key) { - const sk = String(key) - const k = sk.substr(0, 8) + '...' + sk.substr(-8) - return new YAMLSemanticError(source, `The "${k}" key is too long`) -} - -export function resolveComments(collection, comments) { - for (const { afterKey, before, comment } of comments) { - let item = collection.items[before] - if (!item) { - if (comment !== undefined) { - if (collection.comment) collection.comment += '\n' + comment - else collection.comment = comment - } - } else { - if (afterKey && item.value) item = item.value - if (comment === undefined) { - if (afterKey || !item.commentBefore) item.spaceBefore = true - } else { - if (item.commentBefore) item.commentBefore += '\n' + comment - else item.commentBefore = comment - } - } - } -} diff --git a/src/resolve/resolveMap.js b/src/resolve/resolveMap.js deleted file mode 100644 index d3b622b9..00000000 --- a/src/resolve/resolveMap.js +++ /dev/null @@ -1,257 +0,0 @@ -import { Alias } from '../ast/Alias.js' -import { Merge } from '../ast/Merge.js' -import { Pair } from '../ast/Pair.js' -import { YAMLMap } from '../ast/YAMLMap.js' -import { Char, Type } from '../constants.js' -import { PlainValue } from '../cst/PlainValue.js' -import { YAMLSemanticError, YAMLSyntaxError } from '../errors.js' - -import { - checkFlowCollectionEnd, - checkFlowCommentSpace, - getLongKeyError, - resolveComments -} from './collection-utils.js' -import { resolveNode } from './resolveNode.js' - -export function resolveMap(doc, cst) { - const { comments, items } = - cst.type === Type.FLOW_MAP - ? resolveFlowMapItems(doc, cst) - : resolveBlockMapItems(doc, cst) - const map = new YAMLMap(doc.schema) - map.items = items - resolveComments(map, comments) - for (let i = 0; i < items.length; ++i) { - const { key: iKey } = items[i] - if (doc.schema.merge && iKey && iKey.value === Merge.KEY) { - items[i] = new Merge(items[i]) - const sources = items[i].value.items - let error = null - sources.some(node => { - if (node instanceof Alias) { - // During parsing, alias sources are CST nodes; to account for - // circular references their resolved values can't be used here. - const { type } = node.source - if (type === Type.MAP || type === Type.FLOW_MAP) return false - return (error = 'Merge nodes aliases can only point to maps') - } - return (error = 'Merge nodes can only have Alias nodes as values') - }) - if (error) doc.errors.push(new YAMLSemanticError(cst, error)) - } else { - for (let j = i + 1; j < items.length; ++j) { - const { key: jKey } = items[j] - if ( - iKey === jKey || - (iKey && - jKey && - Object.prototype.hasOwnProperty.call(iKey, 'value') && - iKey.value === jKey.value) - ) { - const msg = `Map keys must be unique; "${iKey}" is repeated` - doc.errors.push(new YAMLSemanticError(cst, msg)) - break - } - } - } - } - cst.resolved = map - return map -} - -const valueHasPairComment = ({ context: { lineStart, node, src }, props }) => { - if (props.length === 0) return false - const { start } = props[0] - if (node && start > node.valueRange.start) return false - if (src[start] !== Char.COMMENT) return false - for (let i = lineStart; i < start; ++i) if (src[i] === '\n') return false - return true -} - -function resolvePairComment(item, pair) { - if (!valueHasPairComment(item)) return - const comment = item.getPropValue(0, Char.COMMENT, true) - let found = false - const cb = pair.value.commentBefore - if (cb && cb.startsWith(comment)) { - pair.value.commentBefore = cb.substr(comment.length + 1) - found = true - } else { - const cc = pair.value.comment - if (!item.node && cc && cc.startsWith(comment)) { - pair.value.comment = cc.substr(comment.length + 1) - found = true - } - } - if (found) pair.comment = comment -} - -function resolveBlockMapItems(doc, cst) { - const comments = [] - const items = [] - let key = undefined - let keyStart = null - for (let i = 0; i < cst.items.length; ++i) { - const item = cst.items[i] - switch (item.type) { - case Type.BLANK_LINE: - comments.push({ afterKey: !!key, before: items.length }) - break - case Type.COMMENT: - comments.push({ - afterKey: !!key, - before: items.length, - comment: item.comment - }) - break - case Type.MAP_KEY: - if (key !== undefined) items.push(new Pair(key)) - if (item.error) doc.errors.push(item.error) - key = resolveNode(doc, item.node) - keyStart = null - break - case Type.MAP_VALUE: - { - if (key === undefined) key = null - if (item.error) doc.errors.push(item.error) - if ( - !item.context.atLineStart && - item.node && - item.node.type === Type.MAP && - !item.node.context.atLineStart - ) { - const msg = 'Nested mappings are not allowed in compact mappings' - doc.errors.push(new YAMLSemanticError(item.node, msg)) - } - let valueNode = item.node - if (!valueNode && item.props.length > 0) { - // Comments on an empty mapping value need to be preserved, so we - // need to construct a minimal empty node here to use instead of the - // missing `item.node`. -- eemeli/yaml#19 - valueNode = new PlainValue(Type.PLAIN, []) - valueNode.context = { parent: item, src: item.context.src } - const pos = item.range.start + 1 - valueNode.range = { start: pos, end: pos } - valueNode.valueRange = { start: pos, end: pos } - if (typeof item.range.origStart === 'number') { - const origPos = item.range.origStart + 1 - valueNode.range.origStart = valueNode.range.origEnd = origPos - valueNode.valueRange.origStart = valueNode.valueRange.origEnd = origPos - } - } - const pair = new Pair(key, resolveNode(doc, valueNode)) - resolvePairComment(item, pair) - items.push(pair) - if (key && typeof keyStart === 'number') { - if (item.range.start > keyStart + 1024) - doc.errors.push(getLongKeyError(cst, key)) - } - key = undefined - keyStart = null - } - break - default: - if (key !== undefined) items.push(new Pair(key)) - key = resolveNode(doc, item) - keyStart = item.range.start - if (item.error) doc.errors.push(item.error) - next: for (let j = i + 1; ; ++j) { - const nextItem = cst.items[j] - switch (nextItem && nextItem.type) { - case Type.BLANK_LINE: - case Type.COMMENT: - continue next - case Type.MAP_VALUE: - break next - default: { - const msg = 'Implicit map keys need to be followed by map values' - doc.errors.push(new YAMLSemanticError(item, msg)) - break next - } - } - } - if (item.valueRangeContainsNewline) { - const msg = 'Implicit map keys need to be on a single line' - doc.errors.push(new YAMLSemanticError(item, msg)) - } - } - } - if (key !== undefined) items.push(new Pair(key)) - return { comments, items } -} - -function resolveFlowMapItems(doc, cst) { - const comments = [] - const items = [] - let key = undefined - let explicitKey = false - let next = '{' - for (let i = 0; i < cst.items.length; ++i) { - const item = cst.items[i] - if (typeof item.char === 'string') { - const { char, offset } = item - if (char === '?' && key === undefined && !explicitKey) { - explicitKey = true - next = ':' - continue - } - if (char === ':') { - if (key === undefined) key = null - if (next === ':') { - next = ',' - continue - } - } else { - if (explicitKey) { - if (key === undefined && char !== ',') key = null - explicitKey = false - } - if (key !== undefined) { - items.push(new Pair(key)) - key = undefined - if (char === ',') { - next = ':' - continue - } - } - } - if (char === '}') { - if (i === cst.items.length - 1) continue - } else if (char === next) { - next = ':' - continue - } - const msg = `Flow map contains an unexpected ${char}` - const err = new YAMLSyntaxError(cst, msg) - err.offset = offset - doc.errors.push(err) - } else if (item.type === Type.BLANK_LINE) { - comments.push({ afterKey: !!key, before: items.length }) - } else if (item.type === Type.COMMENT) { - checkFlowCommentSpace(doc.errors, item) - comments.push({ - afterKey: !!key, - before: items.length, - comment: item.comment - }) - } else if (key === undefined) { - if (next === ',') - doc.errors.push( - new YAMLSemanticError(item, 'Separator , missing in flow map') - ) - key = resolveNode(doc, item) - } else { - if (next !== ',') - doc.errors.push( - new YAMLSemanticError(item, 'Indicator : missing in flow map entry') - ) - items.push(new Pair(key, resolveNode(doc, item))) - key = undefined - explicitKey = false - } - } - checkFlowCollectionEnd(doc.errors, cst) - if (key !== undefined) items.push(new Pair(key)) - return { comments, items } -} diff --git a/src/resolve/resolveNode.js b/src/resolve/resolveNode.js deleted file mode 100644 index 3959eb9d..00000000 --- a/src/resolve/resolveNode.js +++ /dev/null @@ -1,145 +0,0 @@ -import { Alias } from '../ast/Alias.js' -import { Char, Type } from '../constants.js' -import { - YAMLReferenceError, - YAMLSemanticError, - YAMLSyntaxError -} from '../errors.js' - -import { resolveScalar } from './resolveScalar.js' -import { resolveTagName } from './resolveTagName.js' -import { resolveTag } from './resolveTag.js' - -const isCollectionItem = node => { - if (!node) return false - const { type } = node - return ( - type === Type.MAP_KEY || type === Type.MAP_VALUE || type === Type.SEQ_ITEM - ) -} - -function resolveNodeProps(errors, node) { - const comments = { before: [], after: [] } - let hasAnchor = false - let hasTag = false - - const props = isCollectionItem(node.context.parent) - ? node.context.parent.props.concat(node.props) - : node.props - for (const { start, end } of props) { - switch (node.context.src[start]) { - case Char.COMMENT: { - if (!node.commentHasRequiredWhitespace(start)) { - const msg = - 'Comments must be separated from other tokens by white space characters' - errors.push(new YAMLSemanticError(node, msg)) - } - const { header, valueRange } = node - const cc = - valueRange && - (start > valueRange.start || (header && start > header.start)) - ? comments.after - : comments.before - cc.push(node.context.src.slice(start + 1, end)) - break - } - - // Actual anchor & tag resolution is handled by schema, here we just complain - case Char.ANCHOR: - if (hasAnchor) { - const msg = 'A node can have at most one anchor' - errors.push(new YAMLSemanticError(node, msg)) - } - hasAnchor = true - break - case Char.TAG: - if (hasTag) { - const msg = 'A node can have at most one tag' - errors.push(new YAMLSemanticError(node, msg)) - } - hasTag = true - break - } - } - return { comments, hasAnchor, hasTag } -} - -function resolveNodeValue(doc, node) { - const { anchors, errors, schema } = doc - - if (node.type === Type.ALIAS) { - const name = node.rawValue - const src = anchors.getNode(name) - if (!src) { - const msg = `Aliased anchor not found: ${name}` - errors.push(new YAMLReferenceError(node, msg)) - return null - } - - // Lazy resolution for circular references - const res = new Alias(src) - anchors._cstAliases.push(res) - return res - } - - const tagName = resolveTagName(doc, node) - if (tagName) return resolveTag(doc, node, tagName) - - if (node.type !== Type.PLAIN) { - const msg = `Failed to resolve ${node.type} node here` - errors.push(new YAMLSyntaxError(node, msg)) - return null - } - - try { - let str = node.strValue || '' - if (typeof str !== 'string') { - str.errors.forEach(error => doc.errors.push(error)) - str = str.str - } - return resolveScalar(str, schema.tags) - } catch (error) { - if (!error.source) error.source = node - errors.push(error) - return null - } -} - -// sets node.resolved on success -export function resolveNode(doc, node) { - if (!node) return null - if (node.error) doc.errors.push(node.error) - - const { comments, hasAnchor, hasTag } = resolveNodeProps(doc.errors, node) - if (hasAnchor) { - const { anchors } = doc - const name = node.anchor - const prev = anchors.getNode(name) - // At this point, aliases for any preceding node with the same anchor - // name have already been resolved, so it may safely be renamed. - if (prev) anchors.map[anchors.newName(name)] = prev - // During parsing, we need to store the CST node in anchors.map as - // anchors need to be available during resolution to allow for - // circular references. - anchors.map[name] = node - } - if (node.type === Type.ALIAS && (hasAnchor || hasTag)) { - const msg = 'An alias node must not specify any properties' - doc.errors.push(new YAMLSemanticError(node, msg)) - } - - const res = resolveNodeValue(doc, node) - if (res) { - res.range = [node.range.start, node.range.end] - if (doc.options.keepCstNodes) res.cstNode = node - if (doc.options.keepNodeTypes) res.type = node.type - const cb = comments.before.join('\n') - if (cb) { - res.commentBefore = res.commentBefore ? `${res.commentBefore}\n${cb}` : cb - } - const ca = comments.after.join('\n') - if (ca) res.comment = res.comment ? `${res.comment}\n${ca}` : ca - } - - return (node.resolved = res) -} diff --git a/src/resolve/resolveScalar.js b/src/resolve/resolveScalar.js deleted file mode 100644 index d2506976..00000000 --- a/src/resolve/resolveScalar.js +++ /dev/null @@ -1,13 +0,0 @@ -import { Scalar } from '../ast/Scalar.js' - -export function resolveScalar(str, tags) { - for (const { format, test, resolve } of tags) { - if (test && test.test(str)) { - let res = resolve(str) - if (!(res instanceof Scalar)) res = new Scalar(res) - if (format) res.format = format - return res - } - } - return new Scalar(str) // fallback to string -} diff --git a/src/resolve/resolveSeq.js b/src/resolve/resolveSeq.js deleted file mode 100644 index dfe643af..00000000 --- a/src/resolve/resolveSeq.js +++ /dev/null @@ -1,139 +0,0 @@ -import { Pair } from '../ast/Pair.js' -import { YAMLSeq } from '../ast/YAMLSeq.js' -import { Type } from '../constants.js' -import { YAMLSemanticError, YAMLSyntaxError } from '../errors.js' - -import { - checkFlowCollectionEnd, - checkFlowCommentSpace, - getLongKeyError, - resolveComments -} from './collection-utils.js' -import { resolveNode } from './resolveNode.js' - -export function resolveSeq(doc, cst) { - const { comments, items } = - cst.type === Type.FLOW_SEQ - ? resolveFlowSeqItems(doc, cst) - : resolveBlockSeqItems(doc, cst) - const seq = new YAMLSeq(doc.schema) - seq.items = items - resolveComments(seq, comments) - cst.resolved = seq - return seq -} - -function resolveBlockSeqItems(doc, cst) { - const comments = [] - const items = [] - for (let i = 0; i < cst.items.length; ++i) { - const item = cst.items[i] - switch (item.type) { - case Type.BLANK_LINE: - comments.push({ before: items.length }) - break - case Type.COMMENT: - comments.push({ comment: item.comment, before: items.length }) - break - case Type.SEQ_ITEM: - if (item.error) doc.errors.push(item.error) - items.push(resolveNode(doc, item.node)) - if (item.hasProps) { - const msg = - 'Sequence items cannot have tags or anchors before the - indicator' - doc.errors.push(new YAMLSemanticError(item, msg)) - } - break - default: - if (item.error) doc.errors.push(item.error) - doc.errors.push( - new YAMLSyntaxError(item, `Unexpected ${item.type} node in sequence`) - ) - } - } - return { comments, items } -} - -function resolveFlowSeqItems(doc, cst) { - const comments = [] - const items = [] - let explicitKey = false - let key = undefined - let keyStart = null - let next = '[' - let prevItem = null - for (let i = 0; i < cst.items.length; ++i) { - const item = cst.items[i] - if (typeof item.char === 'string') { - const { char, offset } = item - if (char !== ':' && (explicitKey || key !== undefined)) { - if (explicitKey && key === undefined) key = next ? items.pop() : null - items.push(new Pair(key)) - explicitKey = false - key = undefined - keyStart = null - } - if (char === next) { - next = null - } else if (!next && char === '?') { - explicitKey = true - } else if (next !== '[' && char === ':' && key === undefined) { - if (next === ',') { - key = items.pop() - if (key instanceof Pair) { - const msg = 'Chaining flow sequence pairs is invalid' - const err = new YAMLSemanticError(cst, msg) - err.offset = offset - doc.errors.push(err) - } - if (!explicitKey && typeof keyStart === 'number') { - const keyEnd = item.range ? item.range.start : item.offset - if (keyEnd > keyStart + 1024) - doc.errors.push(getLongKeyError(cst, key)) - const { src } = prevItem.context - for (let i = keyStart; i < keyEnd; ++i) - if (src[i] === '\n') { - const msg = - 'Implicit keys of flow sequence pairs need to be on a single line' - doc.errors.push(new YAMLSemanticError(prevItem, msg)) - break - } - } - } else { - key = null - } - keyStart = null - explicitKey = false - next = null - } else if (next === '[' || char !== ']' || i < cst.items.length - 1) { - const msg = `Flow sequence contains an unexpected ${char}` - const err = new YAMLSyntaxError(cst, msg) - err.offset = offset - doc.errors.push(err) - } - } else if (item.type === Type.BLANK_LINE) { - comments.push({ before: items.length }) - } else if (item.type === Type.COMMENT) { - checkFlowCommentSpace(doc.errors, item) - comments.push({ comment: item.comment, before: items.length }) - } else { - if (next) { - const msg = `Expected a ${next} in flow sequence` - doc.errors.push(new YAMLSemanticError(item, msg)) - } - const value = resolveNode(doc, item) - if (key === undefined) { - items.push(value) - prevItem = item - } else { - items.push(new Pair(key, value)) - key = undefined - } - keyStart = item.range.start - next = ',' - } - } - checkFlowCollectionEnd(doc.errors, cst) - if (key !== undefined) items.push(new Pair(key)) - return { comments, items } -} diff --git a/src/resolve/resolveTag.js b/src/resolve/resolveTag.js deleted file mode 100644 index df891269..00000000 --- a/src/resolve/resolveTag.js +++ /dev/null @@ -1,95 +0,0 @@ -import { Collection } from '../ast/Collection.js' -import { Scalar } from '../ast/Scalar.js' -import { Type, defaultTags } from '../constants.js' -import { - YAMLReferenceError, - YAMLSemanticError, - YAMLWarning -} from '../errors.js' -import { resolveMap } from './resolveMap.js' -import { resolveScalar } from './resolveScalar.js' -import { resolveSeq } from './resolveSeq.js' - -function resolveByTagName({ knownTags, tags }, tagName, value, onError) { - const matchWithTest = [] - for (const tag of tags) { - if (tag.tag === tagName) { - if (tag.test) { - if (typeof value === 'string') matchWithTest.push(tag) - else onError(`The tag ${tagName} cannot be applied to a collection`) - } else { - const res = tag.resolve(value, onError) - return res instanceof Collection ? res : new Scalar(res) - } - } - } - if (matchWithTest.length > 0) return resolveScalar(value, matchWithTest) - - const kt = knownTags[tagName] - if (kt) { - tags.push(Object.assign({}, kt, { default: false, test: undefined })) - const res = kt.resolve(value, onError) - return res instanceof Collection ? res : new Scalar(res) - } - - return null -} - -export function resolveTag(doc, node, tagName) { - const { MAP, SEQ, STR } = defaultTags - let value, fallback - const onError = message => - doc.errors.push(new YAMLSemanticError(node, message)) - try { - switch (node.type) { - case Type.FLOW_MAP: - case Type.MAP: - value = resolveMap(doc, node) - fallback = MAP - if (tagName === SEQ || tagName === STR) - onError(`The tag ${tagName} cannot be applied to a mapping`) - break - case Type.FLOW_SEQ: - case Type.SEQ: - value = resolveSeq(doc, node) - fallback = SEQ - if (tagName === MAP || tagName === STR) - onError(`The tag ${tagName} cannot be applied to a sequence`) - break - default: - value = node.strValue || '' - if (typeof value !== 'string') { - value.errors.forEach(error => doc.errors.push(error)) - value = value.str - } - if (tagName === MAP || tagName === SEQ) - onError(`The tag ${tagName} cannot be applied to a scalar`) - fallback = STR - } - - const res = resolveByTagName(doc.schema, tagName, value, onError) - if (res) { - if (tagName && node.tag) res.tag = tagName - return res - } - } catch (error) { - /* istanbul ignore if */ - if (!error.source) error.source = node - doc.errors.push(error) - return null - } - - try { - if (!fallback) throw new Error(`The tag ${tagName} is unavailable`) - const msg = `The tag ${tagName} is unavailable, falling back to ${fallback}` - doc.warnings.push(new YAMLWarning(node, msg)) - const res = resolveByTagName(doc.schema, fallback, value, onError) - res.tag = tagName - return res - } catch (error) { - const refError = new YAMLReferenceError(node, error.message) - refError.stack = error.stack - doc.errors.push(refError) - return null - } -} diff --git a/src/resolve/resolveTagName.js b/src/resolve/resolveTagName.js deleted file mode 100644 index 2a105383..00000000 --- a/src/resolve/resolveTagName.js +++ /dev/null @@ -1,75 +0,0 @@ -import { Type, defaultTags } from '../constants.js' -import { YAMLSemanticError, YAMLWarning } from '../errors.js' - -function resolveTagHandle(doc, node) { - const { handle, suffix } = node.tag - let prefix = doc.tagPrefixes.find(p => p.handle === handle) - if (!prefix) { - const dtp = doc.getDefaults().tagPrefixes - if (dtp) prefix = dtp.find(p => p.handle === handle) - if (!prefix) - throw new YAMLSemanticError( - node, - `The ${handle} tag handle is non-default and was not declared.` - ) - } - if (!suffix) - throw new YAMLSemanticError(node, `The ${handle} tag has no suffix.`) - - if (handle === '!' && (doc.version || doc.options.version) === '1.0') { - if (suffix[0] === '^') { - doc.warnings.push( - new YAMLWarning(node, 'YAML 1.0 ^ tag expansion is not supported') - ) - return suffix - } - if (/[:/]/.test(suffix)) { - // word/foo -> tag:word.yaml.org,2002:foo - const vocab = suffix.match(/^([a-z0-9-]+)\/(.*)/i) - return vocab - ? `tag:${vocab[1]}.yaml.org,2002:${vocab[2]}` - : `tag:${suffix}` - } - } - - return prefix.prefix + decodeURIComponent(suffix) -} - -export function resolveTagName(doc, node) { - const { tag, type } = node - let nonSpecific = false - if (tag) { - const { handle, suffix, verbatim } = tag - if (verbatim) { - if (verbatim !== '!' && verbatim !== '!!') return verbatim - const msg = `Verbatim tags aren't resolved, so ${verbatim} is invalid.` - doc.errors.push(new YAMLSemanticError(node, msg)) - } else if (handle === '!' && !suffix) { - nonSpecific = true - } else { - try { - return resolveTagHandle(doc, node) - } catch (error) { - doc.errors.push(error) - } - } - } - - switch (type) { - case Type.BLOCK_FOLDED: - case Type.BLOCK_LITERAL: - case Type.QUOTE_DOUBLE: - case Type.QUOTE_SINGLE: - return defaultTags.STR - case Type.FLOW_MAP: - case Type.MAP: - return defaultTags.MAP - case Type.FLOW_SEQ: - case Type.SEQ: - return defaultTags.SEQ - case Type.PLAIN: - return nonSpecific ? defaultTags.STR : null - default: - return null - } -} From f9451ef5887b5b5d0d44de831101fbad102fc7fe Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 7 Feb 2021 10:12:14 +0200 Subject: [PATCH 79/89] Drop all code in & references to src/cst/ as obsolete --- cst.d.ts | 185 -- index.d.ts | 11 +- src/ast/index.d.ts | 23 +- src/cst/Alias.js | 24 - src/cst/BlankLine.js | 30 - src/cst/BlockValue.js | 209 --- src/cst/Collection.js | 224 --- src/cst/CollectionItem.js | 117 -- src/cst/Comment.js | 24 - src/cst/Directive.js | 43 - src/cst/Document.js | 227 --- src/cst/FlowCollection.js | 175 -- src/cst/Node.js | 300 --- src/cst/ParseContext.js | 228 --- src/cst/PlainValue.js | 146 -- src/cst/QuoteDouble.js | 182 -- src/cst/QuoteSingle.js | 95 - src/cst/README.md | 3 - src/cst/Range.js | 45 - src/cst/index.d.ts | 182 -- src/cst/parse.js | 32 - src/cst/source-utils.js | 132 -- src/doc/Document.d.ts | 7 +- src/doc/Schema.d.ts | 2 - src/errors.d.ts | 20 +- src/errors.js | 49 +- tests/cst/Node.js | 32 - tests/cst/YAML-1.2.spec.js | 3608 ------------------------------------ tests/cst/common.js | 51 - tests/cst/corner-cases.js | 585 ------ tests/cst/parse.js | 307 --- tests/cst/set-value.js | 53 - tests/cst/source-utils.js | 49 - tests/doc/comments.js | 13 - tests/doc/errors.js | 27 +- types.d.ts | 25 +- util.d.ts | 9 +- 37 files changed, 74 insertions(+), 7400 deletions(-) delete mode 100644 cst.d.ts delete mode 100644 src/cst/Alias.js delete mode 100644 src/cst/BlankLine.js delete mode 100644 src/cst/BlockValue.js delete mode 100644 src/cst/Collection.js delete mode 100644 src/cst/CollectionItem.js delete mode 100644 src/cst/Comment.js delete mode 100644 src/cst/Directive.js delete mode 100644 src/cst/Document.js delete mode 100644 src/cst/FlowCollection.js delete mode 100644 src/cst/Node.js delete mode 100644 src/cst/ParseContext.js delete mode 100644 src/cst/PlainValue.js delete mode 100644 src/cst/QuoteDouble.js delete mode 100644 src/cst/QuoteSingle.js delete mode 100644 src/cst/README.md delete mode 100644 src/cst/Range.js delete mode 100644 src/cst/index.d.ts delete mode 100644 src/cst/parse.js delete mode 100644 src/cst/source-utils.js delete mode 100644 tests/cst/Node.js delete mode 100644 tests/cst/YAML-1.2.spec.js delete mode 100644 tests/cst/common.js delete mode 100644 tests/cst/corner-cases.js delete mode 100644 tests/cst/parse.js delete mode 100644 tests/cst/set-value.js delete mode 100644 tests/cst/source-utils.js diff --git a/cst.d.ts b/cst.d.ts deleted file mode 100644 index 9f2d4408..00000000 --- a/cst.d.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { Type, YAMLSyntaxError } from './util' - -export namespace CST { - interface Range { - start: number - end: number - origStart?: number - origEnd?: number - isEmpty(): boolean - } - - interface ParseContext { - /** Node starts at beginning of line */ - atLineStart: boolean - /** true if currently in a collection context */ - inCollection: boolean - /** true if currently in a flow context */ - inFlow: boolean - /** Current level of indentation */ - indent: number - /** Start of the current line */ - lineStart: number - /** The parent of the node */ - parent: Node - /** Source of the YAML document */ - src: string - } - - interface Node { - context: ParseContext | null - /** if not null, indicates a parser failure */ - error: YAMLSyntaxError | null - /** span of context.src parsed into this node */ - range: Range | null - valueRange: Range | null - /** anchors, tags and comments */ - props: Range[] - /** specific node type */ - type: Type - /** if non-null, overrides source value */ - value: string | null - - readonly anchor: string | null - readonly comment: string | null - readonly hasComment: boolean - readonly hasProps: boolean - readonly jsonLike: boolean - readonly rangeAsLinePos: null | { - start: { line: number; col: number } - end?: { line: number; col: number } - } - readonly rawValue: string | null - readonly tag: - | null - | { verbatim: string } - | { handle: string; suffix: string } - readonly valueRangeContainsNewline: boolean - } - - interface Alias extends Node { - type: Type.ALIAS - /** contain the anchor without the * prefix */ - readonly rawValue: string - } - - type Scalar = BlockValue | PlainValue | QuoteValue - - interface BlockValue extends Node { - type: Type.BLOCK_FOLDED | Type.BLOCK_LITERAL - chomping: 'CLIP' | 'KEEP' | 'STRIP' - blockIndent: number | null - header: Range - readonly strValue: string | null - } - - interface BlockFolded extends BlockValue { - type: Type.BLOCK_FOLDED - } - - interface BlockLiteral extends BlockValue { - type: Type.BLOCK_LITERAL - } - - interface PlainValue extends Node { - type: Type.PLAIN - readonly strValue: string | null - } - - interface QuoteValue extends Node { - type: Type.QUOTE_DOUBLE | Type.QUOTE_SINGLE - readonly strValue: - | null - | string - | { str: string; errors: YAMLSyntaxError[] } - } - - interface QuoteDouble extends QuoteValue { - type: Type.QUOTE_DOUBLE - } - - interface QuoteSingle extends QuoteValue { - type: Type.QUOTE_SINGLE - } - - interface Comment extends Node { - type: Type.COMMENT - readonly anchor: null - readonly comment: string - readonly rawValue: null - readonly tag: null - } - - interface BlankLine extends Node { - type: Type.BLANK_LINE - } - - interface MapItem extends Node { - type: Type.MAP_KEY | Type.MAP_VALUE - node: ContentNode | null - } - - interface MapKey extends MapItem { - type: Type.MAP_KEY - } - - interface MapValue extends MapItem { - type: Type.MAP_VALUE - } - - interface Map extends Node { - type: Type.MAP - /** implicit keys are not wrapped */ - items: Array - } - - interface SeqItem extends Node { - type: Type.SEQ_ITEM - node: ContentNode | null - } - - interface Seq extends Node { - type: Type.SEQ - items: Array - } - - interface FlowChar { - char: '{' | '}' | '[' | ']' | ',' | '?' | ':' - offset: number - origOffset?: number - } - - interface FlowCollection extends Node { - type: Type.FLOW_MAP | Type.FLOW_SEQ - items: Array< - FlowChar | BlankLine | Comment | Alias | Scalar | FlowCollection - > - } - - interface FlowMap extends FlowCollection { - type: Type.FLOW_MAP - } - - interface FlowSeq extends FlowCollection { - type: Type.FLOW_SEQ - } - - type ContentNode = Alias | Scalar | Map | Seq | FlowCollection - - interface Directive extends Node { - type: Type.DIRECTIVE - name: string - readonly anchor: null - readonly parameters: string[] - readonly tag: null - } - - interface Document extends Node { - type: Type.DOCUMENT - directives: Array - contents: Array - readonly anchor: null - readonly comment: null - readonly tag: null - } -} diff --git a/index.d.ts b/index.d.ts index f7e8a41b..cd9ad4c7 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,4 +1,3 @@ -import { CST } from './cst' import { AST, Alias, @@ -13,13 +12,7 @@ import { } from './types' import { Type, YAMLError, YAMLWarning } from './util' -export { AST, CST } - -export function parseCST(str: string): ParsedCST - -export interface ParsedCST extends Array { - setOrigRanges(): boolean -} +export { AST } /** * Apply a visitor to an AST node or document. @@ -294,7 +287,7 @@ export interface CreateNodeOptions { } export class Document extends Collection { - cstNode?: CST.Document + // cstNode?: CST.Document /** * @param value - The initial value for the document, which will be wrapped * in a Node container. diff --git a/src/ast/index.d.ts b/src/ast/index.d.ts index e6868f35..4f4a699d 100644 --- a/src/ast/index.d.ts +++ b/src/ast/index.d.ts @@ -1,5 +1,4 @@ import { Type } from '../constants' -import { CST } from '../cst' import { Schema } from '../doc/Schema' export class Node { @@ -8,7 +7,7 @@ export class Node { /** A comment before this */ commentBefore?: string | null /** Only available when `keepCstNodes` is set to `true` */ - cstNode?: CST.Node + // cstNode?: CST.Node /** * The [start, end] range of characters of the source parsed * into this node (undefined for pairs or if not parsed) @@ -60,7 +59,7 @@ export class Alias extends Node { constructor(source: Node) type: Type.ALIAS source: Node - cstNode?: CST.Alias + // cstNode?: CST.Alias toString(ctx: Schema.StringifyContext): string } @@ -198,48 +197,48 @@ export namespace AST { interface BlockFolded extends Scalar { type: Type.BLOCK_FOLDED - cstNode?: CST.BlockFolded + // cstNode?: CST.BlockFolded } interface BlockLiteral extends Scalar { type: Type.BLOCK_LITERAL - cstNode?: CST.BlockLiteral + // cstNode?: CST.BlockLiteral } interface PlainValue extends Scalar { type: Type.PLAIN - cstNode?: CST.PlainValue + // cstNode?: CST.PlainValue } interface QuoteDouble extends Scalar { type: Type.QUOTE_DOUBLE - cstNode?: CST.QuoteDouble + // cstNode?: CST.QuoteDouble } interface QuoteSingle extends Scalar { type: Type.QUOTE_SINGLE - cstNode?: CST.QuoteSingle + // cstNode?: CST.QuoteSingle } interface FlowMap extends YAMLMap { type: Type.FLOW_MAP - cstNode?: CST.FlowMap + // cstNode?: CST.FlowMap } interface BlockMap extends YAMLMap { type: Type.MAP - cstNode?: CST.Map + // cstNode?: CST.Map } interface FlowSeq extends YAMLSeq { type: Type.FLOW_SEQ items: Array - cstNode?: CST.FlowSeq + // cstNode?: CST.FlowSeq } interface BlockSeq extends YAMLSeq { type: Type.SEQ items: Array - cstNode?: CST.Seq + // cstNode?: CST.Seq } } diff --git a/src/cst/Alias.js b/src/cst/Alias.js deleted file mode 100644 index 74994858..00000000 --- a/src/cst/Alias.js +++ /dev/null @@ -1,24 +0,0 @@ -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class Alias extends Node { - /** - * Parses an *alias from the source - * - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this scalar - */ - parse(context, start) { - this.context = context - const { src } = context - let offset = Node.endOfIdentifier(src, start + 1) - this.valueRange = new Range(start + 1, offset) - offset = Node.endOfWhiteSpace(src, offset) - offset = this.parseComment(offset) - trace: this.type, - { valueRange: this.valueRange, comment: this.comment }, - JSON.stringify(this.rawValue) - return offset - } -} diff --git a/src/cst/BlankLine.js b/src/cst/BlankLine.js deleted file mode 100644 index 45d1f12c..00000000 --- a/src/cst/BlankLine.js +++ /dev/null @@ -1,30 +0,0 @@ -import { Type } from '../constants.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class BlankLine extends Node { - constructor() { - super(Type.BLANK_LINE) - } - - /* istanbul ignore next */ - get includesTrailingLines() { - // This is never called from anywhere, but if it were, - // this is the value it should return. - return true - } - - /** - * Parses a blank line from the source - * - * @param {ParseContext} context - * @param {number} start - Index of first \n character - * @returns {number} - Index of the character after this - */ - parse(context, start) { - this.context = context - this.range = new Range(start, start + 1) - trace: this.type, this.range - return start + 1 - } -} diff --git a/src/cst/BlockValue.js b/src/cst/BlockValue.js deleted file mode 100644 index fd8aedc8..00000000 --- a/src/cst/BlockValue.js +++ /dev/null @@ -1,209 +0,0 @@ -import { Type } from '../constants.js' -import { YAMLSemanticError } from '../errors.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export const Chomp = { - CLIP: 'CLIP', - KEEP: 'KEEP', - STRIP: 'STRIP' -} - -export class BlockValue extends Node { - constructor(type, props) { - super(type, props) - this.blockIndent = null - this.chomping = Chomp.CLIP - this.header = null - } - - get includesTrailingLines() { - return this.chomping === Chomp.KEEP - } - - get strValue() { - if (!this.valueRange || !this.context) return null - let { start, end } = this.valueRange - const { indent, src } = this.context - if (this.valueRange.isEmpty()) return '' - let lastNewLine = null - let ch = src[end - 1] - while (ch === '\n' || ch === '\t' || ch === ' ') { - end -= 1 - if (end <= start) { - if (this.chomping === Chomp.KEEP) break - else return '' // probably never happens - } - if (ch === '\n') lastNewLine = end - ch = src[end - 1] - } - let keepStart = end + 1 - if (lastNewLine) { - if (this.chomping === Chomp.KEEP) { - keepStart = lastNewLine - end = this.valueRange.end - } else { - end = lastNewLine - } - } - const bi = indent + this.blockIndent - const folded = this.type === Type.BLOCK_FOLDED - let atStart = true - let str = '' - let sep = '' - let prevMoreIndented = false - for (let i = start; i < end; ++i) { - for (let j = 0; j < bi; ++j) { - if (src[i] !== ' ') break - i += 1 - } - const ch = src[i] - if (ch === '\n') { - if (sep === '\n') str += '\n' - else sep = '\n' - } else { - const lineEnd = Node.endOfLine(src, i) - const line = src.slice(i, lineEnd) - i = lineEnd - if (folded && (ch === ' ' || ch === '\t') && i < keepStart) { - if (sep === ' ') sep = '\n' - else if (!prevMoreIndented && !atStart && sep === '\n') sep = '\n\n' - str += sep + line //+ ((lineEnd < end && src[lineEnd]) || '') - sep = (lineEnd < end && src[lineEnd]) || '' - prevMoreIndented = true - } else { - str += sep + line - sep = folded && i < keepStart ? ' ' : '\n' - prevMoreIndented = false - } - if (atStart && line !== '') atStart = false - } - } - return this.chomping === Chomp.STRIP ? str : str + '\n' - } - - parseBlockHeader(start) { - const { src } = this.context - let offset = start + 1 - let bi = '' - while (true) { - const ch = src[offset] - switch (ch) { - case '-': - this.chomping = Chomp.STRIP - break - case '+': - this.chomping = Chomp.KEEP - break - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - bi += ch - break - default: - this.blockIndent = Number(bi) || null - this.header = new Range(start, offset) - return offset - } - offset += 1 - } - } - - parseBlockValue(start) { - const { indent, src } = this.context - const explicit = !!this.blockIndent - let offset = start - let valueEnd = start - let minBlockIndent = 1 - for (let ch = src[offset]; ch === '\n'; ch = src[offset]) { - offset += 1 - if (Node.atDocumentBoundary(src, offset)) break - const end = Node.endOfBlockIndent(src, indent, offset) // should not include tab? - if (end === null) break - const ch = src[end] - const lineIndent = end - (offset + indent) - if (!this.blockIndent) { - // no explicit block indent, none yet detected - if (src[end] !== '\n') { - // first line with non-whitespace content - if (lineIndent < minBlockIndent) { - const msg = - 'Block scalars with more-indented leading empty lines must use an explicit indentation indicator' - this.error = new YAMLSemanticError(this, msg) - } - this.blockIndent = lineIndent - } else if (lineIndent > minBlockIndent) { - // empty line with more whitespace - minBlockIndent = lineIndent - } - } else if (ch && ch !== '\n' && lineIndent < this.blockIndent) { - if (src[end] === '#') break - if (!this.error) { - const src = explicit ? 'explicit indentation indicator' : 'first line' - const msg = `Block scalars must not be less indented than their ${src}` - this.error = new YAMLSemanticError(this, msg) - } - } - if (src[end] === '\n') { - offset = end - } else { - offset = valueEnd = Node.endOfLine(src, end) - } - } - if (this.chomping !== Chomp.KEEP) { - offset = src[valueEnd] ? valueEnd + 1 : valueEnd - } - this.valueRange = new Range(start + 1, offset) - return offset - } - - /** - * Parses a block value from the source - * - * Accepted forms are: - * ``` - * BS - * block - * lines - * - * BS #comment - * block - * lines - * ``` - * where the block style BS matches the regexp `[|>][-+1-9]*` and block lines - * are empty or have an indent level greater than `indent`. - * - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this block - */ - parse(context, start) { - this.context = context - trace: 'block-start', context.pretty, { start } - const { src } = context - let offset = this.parseBlockHeader(start) - offset = Node.endOfWhiteSpace(src, offset) - offset = this.parseComment(offset) - offset = this.parseBlockValue(offset) - trace: this.type, - { - style: this.blockStyle, - valueRange: this.valueRange, - comment: this.comment - }, - JSON.stringify(this.rawValue) - return offset - } - - setOrigRanges(cr, offset) { - offset = super.setOrigRanges(cr, offset) - return this.header ? this.header.setOrigRange(cr, offset) : offset - } -} diff --git a/src/cst/Collection.js b/src/cst/Collection.js deleted file mode 100644 index 334a1a5d..00000000 --- a/src/cst/Collection.js +++ /dev/null @@ -1,224 +0,0 @@ -import { Type } from '../constants.js' -import { YAMLSyntaxError } from '../errors.js' -import { BlankLine } from './BlankLine.js' -import { CollectionItem } from './CollectionItem.js' -import { Comment } from './Comment.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export function grabCollectionEndComments(node) { - let cnode = node - while (cnode instanceof CollectionItem) cnode = cnode.node - if (!(cnode instanceof Collection)) return null - const len = cnode.items.length - let ci = -1 - for (let i = len - 1; i >= 0; --i) { - const n = cnode.items[i] - if (n.type === Type.COMMENT) { - // Keep sufficiently indented comments with preceding node - const { indent, lineStart } = n.context - if (indent > 0 && n.range.start >= lineStart + indent) break - ci = i - } else if (n.type === Type.BLANK_LINE) ci = i - else break - } - if (ci === -1) return null - const ca = cnode.items.splice(ci, len - ci) - trace: 'item-end-comments', ca - const prevEnd = ca[0].range.start - while (true) { - cnode.range.end = prevEnd - if (cnode.valueRange && cnode.valueRange.end > prevEnd) - cnode.valueRange.end = prevEnd - if (cnode === node) break - cnode = cnode.context.parent - } - return ca -} - -export class Collection extends Node { - static nextContentHasIndent(src, offset, indent) { - const lineStart = Node.endOfLine(src, offset) + 1 - offset = Node.endOfWhiteSpace(src, lineStart) - const ch = src[offset] - if (!ch) return false - if (offset >= lineStart + indent) return true - if (ch !== '#' && ch !== '\n') return false - return Collection.nextContentHasIndent(src, offset, indent) - } - - constructor(firstItem) { - super(firstItem.type === Type.SEQ_ITEM ? Type.SEQ : Type.MAP) - for (let i = firstItem.props.length - 1; i >= 0; --i) { - if (firstItem.props[i].start < firstItem.context.lineStart) { - // props on previous line are assumed by the collection - this.props = firstItem.props.slice(0, i + 1) - firstItem.props = firstItem.props.slice(i + 1) - const itemRange = firstItem.props[0] || firstItem.valueRange - firstItem.range.start = itemRange.start - break - } - } - this.items = [firstItem] - const ec = grabCollectionEndComments(firstItem) - if (ec) Array.prototype.push.apply(this.items, ec) - } - - get includesTrailingLines() { - return this.items.length > 0 - } - - /** - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this - */ - parse(context, start) { - trace: 'collection-start', context.pretty, { start } - this.context = context - const { parseNode, src } = context - // It's easier to recalculate lineStart here rather than tracking down the - // last context from which to read it -- eemeli/yaml#2 - let lineStart = Node.startOfLine(src, start) - const firstItem = this.items[0] - // First-item context needs to be correct for later comment handling - // -- eemeli/yaml#17 - firstItem.context.parent = this - this.valueRange = Range.copy(firstItem.valueRange) - const indent = firstItem.range.start - firstItem.context.lineStart - let offset = start - offset = Node.normalizeOffset(src, offset) - let ch = src[offset] - let atLineStart = Node.endOfWhiteSpace(src, lineStart) === offset - let prevIncludesTrailingLines = false - trace: 'items-start', { offset, indent, lineStart, ch: JSON.stringify(ch) } - while (ch) { - while (ch === '\n' || ch === '#') { - if (atLineStart && ch === '\n' && !prevIncludesTrailingLines) { - const blankLine = new BlankLine() - offset = blankLine.parse({ src }, offset) - this.valueRange.end = offset - if (offset >= src.length) { - ch = null - break - } - this.items.push(blankLine) - trace: 'collection-blankline', blankLine.range - offset -= 1 // blankLine.parse() consumes terminal newline - } else if (ch === '#') { - if ( - offset < lineStart + indent && - !Collection.nextContentHasIndent(src, offset, indent) - ) { - trace: 'end:comment-unindent', { offset, lineStart, indent } - return offset - } - const comment = new Comment() - offset = comment.parse({ indent, lineStart, src }, offset) - this.items.push(comment) - this.valueRange.end = offset - if (offset >= src.length) { - ch = null - break - } - } - lineStart = offset + 1 - offset = Node.endOfIndent(src, lineStart) - if (Node.atBlank(src, offset)) { - const wsEnd = Node.endOfWhiteSpace(src, offset) - const next = src[wsEnd] - if (!next || next === '\n' || next === '#') { - offset = wsEnd - } - } - ch = src[offset] - atLineStart = true - } - if (!ch) { - trace: 'end:src', { offset } - break - } - if (offset !== lineStart + indent && (atLineStart || ch !== ':')) { - if (offset < lineStart + indent) { - trace: 'end:unindent', - { offset, lineStart, indent, ch: JSON.stringify(ch) } - if (lineStart > start) offset = lineStart - break - } else if (!this.error) { - const msg = 'All collection items must start at the same column' - this.error = new YAMLSyntaxError(this, msg) - } - } - if (firstItem.type === Type.SEQ_ITEM) { - if (ch !== '-') { - trace: 'end:typeswitch', - { offset, lineStart, indent, ch: JSON.stringify(ch) } - if (lineStart > start) offset = lineStart - break - } - } else if (ch === '-' && !this.error) { - // map key may start with -, as long as it's followed by a non-whitespace char - const next = src[offset + 1] - if (!next || next === '\n' || next === '\t' || next === ' ') { - const msg = 'A collection cannot be both a mapping and a sequence' - this.error = new YAMLSyntaxError(this, msg) - } - } - trace: 'item-start', this.items.length, { ch: JSON.stringify(ch) } - const node = parseNode( - { atLineStart, inCollection: true, indent, lineStart, parent: this }, - offset - ) - if (!node) return offset // at next document start - this.items.push(node) - this.valueRange.end = node.valueRange.end - offset = Node.normalizeOffset(src, node.range.end) - ch = src[offset] - atLineStart = false - prevIncludesTrailingLines = node.includesTrailingLines - // Need to reset lineStart and atLineStart here if preceding node's range - // has advanced to check the current line's indentation level - // -- eemeli/yaml#10 & eemeli/yaml#38 - if (ch) { - let ls = offset - 1 - let prev = src[ls] - while (prev === ' ' || prev === '\t') prev = src[--ls] - if (prev === '\n') { - lineStart = ls + 1 - atLineStart = true - } - } - const ec = grabCollectionEndComments(node) - if (ec) Array.prototype.push.apply(this.items, ec) - trace: 'item-end', node.type, { offset, ch: JSON.stringify(ch) } - } - trace: 'items', this.items - return offset - } - - setOrigRanges(cr, offset) { - offset = super.setOrigRanges(cr, offset) - this.items.forEach(node => { - offset = node.setOrigRanges(cr, offset) - }) - return offset - } - - toString() { - const { - context: { src }, - items, - range, - value - } = this - if (value != null) return value - let str = src.slice(range.start, items[0].range.start) + String(items[0]) - for (let i = 1; i < items.length; ++i) { - const item = items[i] - const { atLineStart, indent } = item.context - if (atLineStart) for (let i = 0; i < indent; ++i) str += ' ' - str += String(item) - } - return Node.addStringTerminator(src, range.end, str) - } -} diff --git a/src/cst/CollectionItem.js b/src/cst/CollectionItem.js deleted file mode 100644 index 7ddbcb47..00000000 --- a/src/cst/CollectionItem.js +++ /dev/null @@ -1,117 +0,0 @@ -import { Type } from '../constants.js' -import { YAMLSemanticError } from '../errors.js' -import { BlankLine } from './BlankLine.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class CollectionItem extends Node { - constructor(type, props) { - super(type, props) - this.node = null - } - - get includesTrailingLines() { - return !!this.node && this.node.includesTrailingLines - } - - /** - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this - */ - parse(context, start) { - this.context = context - trace: 'item-start', context.pretty, { start } - const { parseNode, src } = context - let { atLineStart, lineStart } = context - if (!atLineStart && this.type === Type.SEQ_ITEM) - this.error = new YAMLSemanticError( - this, - 'Sequence items must not have preceding content on the same line' - ) - const indent = atLineStart ? start - lineStart : context.indent - let offset = Node.endOfWhiteSpace(src, start + 1) - let ch = src[offset] - const inlineComment = ch === '#' - const comments = [] - let blankLine = null - while (ch === '\n' || ch === '#') { - if (ch === '#') { - const end = Node.endOfLine(src, offset + 1) - comments.push(new Range(offset, end)) - offset = end - } else { - atLineStart = true - lineStart = offset + 1 - const wsEnd = Node.endOfWhiteSpace(src, lineStart) - if (src[wsEnd] === '\n' && comments.length === 0) { - blankLine = new BlankLine() - lineStart = blankLine.parse({ src }, lineStart) - } - offset = Node.endOfIndent(src, lineStart) - } - ch = src[offset] - } - trace: 'item-parse?', - { - indentDiff: offset - (lineStart + indent), - ch: ch && JSON.stringify(ch) - } - if ( - Node.nextNodeIsIndented( - ch, - offset - (lineStart + indent), - this.type !== Type.SEQ_ITEM - ) - ) { - this.node = parseNode( - { atLineStart, inCollection: false, indent, lineStart, parent: this }, - offset - ) - } else if (ch && lineStart > start + 1) { - offset = lineStart - 1 - } - if (this.node) { - if (blankLine) { - // Only blank lines preceding non-empty nodes are captured. Note that - // this means that collection item range start indices do not always - // increase monotonically. -- eemeli/yaml#126 - const items = context.parent.items || context.parent.contents - if (items) items.push(blankLine) - } - if (comments.length) Array.prototype.push.apply(this.props, comments) - offset = this.node.range.end - } else { - if (inlineComment) { - const c = comments[0] - this.props.push(c) - offset = c.end - } else { - offset = Node.endOfLine(src, start + 1) - } - } - const end = this.node ? this.node.valueRange.end : offset - trace: 'item-end', { start, end, offset } - this.valueRange = new Range(start, end) - return offset - } - - setOrigRanges(cr, offset) { - offset = super.setOrigRanges(cr, offset) - return this.node ? this.node.setOrigRanges(cr, offset) : offset - } - - toString() { - const { - context: { src }, - node, - range, - value - } = this - if (value != null) return value - const str = node - ? src.slice(range.start, node.range.start) + String(node) - : src.slice(range.start, range.end) - return Node.addStringTerminator(src, range.end, str) - } -} diff --git a/src/cst/Comment.js b/src/cst/Comment.js deleted file mode 100644 index 14559ae7..00000000 --- a/src/cst/Comment.js +++ /dev/null @@ -1,24 +0,0 @@ -import { Type } from '../constants.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class Comment extends Node { - constructor() { - super(Type.COMMENT) - } - - /** - * Parses a comment line from the source - * - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this scalar - */ - parse(context, start) { - this.context = context - const offset = this.parseComment(start) - this.range = new Range(start, offset) - trace: this.type, this.range, this.comment - return offset - } -} diff --git a/src/cst/Directive.js b/src/cst/Directive.js deleted file mode 100644 index b7ac5e83..00000000 --- a/src/cst/Directive.js +++ /dev/null @@ -1,43 +0,0 @@ -import { Type } from '../constants.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class Directive extends Node { - constructor() { - super(Type.DIRECTIVE) - this.name = null - } - - get parameters() { - const raw = this.rawValue - return raw ? raw.trim().split(/[ \t]+/) : [] - } - - parseName(start) { - const { src } = this.context - let offset = start - let ch = src[offset] - while (ch && ch !== '\n' && ch !== '\t' && ch !== ' ') - ch = src[(offset += 1)] - this.name = src.slice(start, offset) - return offset - } - - parseParameters(start) { - const { src } = this.context - let offset = start - let ch = src[offset] - while (ch && ch !== '\n' && ch !== '#') ch = src[(offset += 1)] - this.valueRange = new Range(start, offset) - return offset - } - - parse(context, start) { - this.context = context - let offset = this.parseName(start + 1) - offset = this.parseParameters(offset) - offset = this.parseComment(offset) - this.range = new Range(start, offset) - return offset - } -} diff --git a/src/cst/Document.js b/src/cst/Document.js deleted file mode 100644 index 969622e7..00000000 --- a/src/cst/Document.js +++ /dev/null @@ -1,227 +0,0 @@ -import { Char, Type } from '../constants.js' -import { YAMLSemanticError, YAMLSyntaxError } from '../errors.js' -import { BlankLine } from './BlankLine.js' -import { grabCollectionEndComments } from './Collection.js' -import { Comment } from './Comment.js' -import { Directive } from './Directive.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class Document extends Node { - static startCommentOrEndBlankLine(src, start) { - const offset = Node.endOfWhiteSpace(src, start) - const ch = src[offset] - return ch === '#' || ch === '\n' ? offset : start - } - - constructor() { - super(Type.DOCUMENT) - this.directives = null - this.contents = null - this.directivesEndMarker = null - this.documentEndMarker = null - } - - parseDirectives(start) { - const { src } = this.context - this.directives = [] - let atLineStart = true - let hasDirectives = false - let offset = start - while (!Node.atDocumentBoundary(src, offset, Char.DIRECTIVES_END)) { - offset = Document.startCommentOrEndBlankLine(src, offset) - switch (src[offset]) { - case '\n': - if (atLineStart) { - const blankLine = new BlankLine() - offset = blankLine.parse({ src }, offset) - if (offset < src.length) { - this.directives.push(blankLine) - trace: 'directive-blankline', blankLine.range - } - } else { - offset += 1 - atLineStart = true - } - break - case '#': - { - const comment = new Comment() - offset = comment.parse({ src }, offset) - this.directives.push(comment) - atLineStart = false - trace: 'directive-comment', comment.comment - } - break - case '%': - { - const directive = new Directive() - offset = directive.parse({ parent: this, src }, offset) - this.directives.push(directive) - hasDirectives = true - atLineStart = false - trace: 'directive', - { valueRange: directive.valueRange, comment: directive.comment }, - JSON.stringify(directive.rawValue) - } - break - default: - if (hasDirectives) { - this.error = new YAMLSemanticError( - this, - 'Missing directives-end indicator line' - ) - } else if (this.directives.length > 0) { - this.contents = this.directives - this.directives = [] - } - return offset - } - } - if (src[offset]) { - this.directivesEndMarker = new Range(offset, offset + 3) - return offset + 3 - } - if (hasDirectives) { - this.error = new YAMLSemanticError( - this, - 'Missing directives-end indicator line' - ) - } else if (this.directives.length > 0) { - this.contents = this.directives - this.directives = [] - } - return offset - } - - parseContents(start) { - const { parseNode, src } = this.context - if (!this.contents) this.contents = [] - let lineStart = start - while (src[lineStart - 1] === '-') lineStart -= 1 - let offset = Node.endOfWhiteSpace(src, start) - let atLineStart = lineStart === start - this.valueRange = new Range(offset) - while (!Node.atDocumentBoundary(src, offset, Char.DOCUMENT_END)) { - switch (src[offset]) { - case '\n': - if (atLineStart) { - const blankLine = new BlankLine() - offset = blankLine.parse({ src }, offset) - if (offset < src.length) { - this.contents.push(blankLine) - trace: 'content-blankline', blankLine.range - } - } else { - offset += 1 - atLineStart = true - } - lineStart = offset - break - case '#': - { - const comment = new Comment() - offset = comment.parse({ src }, offset) - this.contents.push(comment) - trace: 'content-comment', comment.comment - atLineStart = false - } - break - default: { - const iEnd = Node.endOfIndent(src, offset) - const context = { - atLineStart, - indent: -1, - inFlow: false, - inCollection: false, - lineStart, - parent: this - } - const node = parseNode(context, iEnd) - if (!node) return (this.valueRange.end = iEnd) // at next document start - this.contents.push(node) - offset = node.range.end - atLineStart = false - const ec = grabCollectionEndComments(node) - if (ec) Array.prototype.push.apply(this.contents, ec) - trace: 'content-node', - { valueRange: node.valueRange, comment: node.comment }, - JSON.stringify(node.rawValue) - } - } - offset = Document.startCommentOrEndBlankLine(src, offset) - } - this.valueRange.end = offset - if (src[offset]) { - this.documentEndMarker = new Range(offset, offset + 3) - offset += 3 - if (src[offset]) { - offset = Node.endOfWhiteSpace(src, offset) - if (src[offset] === '#') { - const comment = new Comment() - offset = comment.parse({ src }, offset) - this.contents.push(comment) - trace: 'document-suffix-comment', comment.comment - } - switch (src[offset]) { - case '\n': - offset += 1 - break - case undefined: - break - default: - this.error = new YAMLSyntaxError( - this, - 'Document end marker line cannot have a non-comment suffix' - ) - } - } - } - return offset - } - - /** - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this - */ - parse(context, start) { - context.root = this - this.context = context - const { src } = context - trace: 'DOC START', JSON.stringify(src.slice(start)) - let offset = src.charCodeAt(start) === 0xfeff ? start + 1 : start // skip BOM - offset = this.parseDirectives(offset) - offset = this.parseContents(offset) - trace: 'DOC', this.contents - return offset - } - - setOrigRanges(cr, offset) { - offset = super.setOrigRanges(cr, offset) - this.directives.forEach(node => { - offset = node.setOrigRanges(cr, offset) - }) - if (this.directivesEndMarker) - offset = this.directivesEndMarker.setOrigRange(cr, offset) - this.contents.forEach(node => { - offset = node.setOrigRanges(cr, offset) - }) - if (this.documentEndMarker) - offset = this.documentEndMarker.setOrigRange(cr, offset) - return offset - } - - toString() { - const { contents, directives, value } = this - if (value != null) return value - let str = directives.join('') - if (contents.length > 0) { - if (directives.length > 0 || contents[0].type === Type.COMMENT) - str += '---\n' - str += contents.join('') - } - if (str[str.length - 1] !== '\n') str += '\n' - return str - } -} diff --git a/src/cst/FlowCollection.js b/src/cst/FlowCollection.js deleted file mode 100644 index f744bb53..00000000 --- a/src/cst/FlowCollection.js +++ /dev/null @@ -1,175 +0,0 @@ -import { Type } from '../constants.js' -import { YAMLSemanticError } from '../errors.js' -import { BlankLine } from './BlankLine.js' -import { Comment } from './Comment.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class FlowCollection extends Node { - constructor(type, props) { - super(type, props) - this.items = null - } - - prevNodeIsJsonLike(idx = this.items.length) { - const node = this.items[idx - 1] - return ( - !!node && - (node.jsonLike || - (node.type === Type.COMMENT && this.prevNodeIsJsonLike(idx - 1))) - ) - } - - /** - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this - */ - parse(context, start) { - trace: 'flow-start', context.pretty, { start } - this.context = context - const { parseNode, src } = context - let { indent, lineStart } = context - let char = src[start] // { or [ - this.items = [{ char, offset: start }] - let offset = Node.endOfWhiteSpace(src, start + 1) - char = src[offset] - while (char && char !== ']' && char !== '}') { - trace: 'item-start', this.items.length, char - switch (char) { - case '\n': - { - lineStart = offset + 1 - const wsEnd = Node.endOfWhiteSpace(src, lineStart) - if (src[wsEnd] === '\n') { - const blankLine = new BlankLine() - lineStart = blankLine.parse({ src }, lineStart) - this.items.push(blankLine) - } - offset = Node.endOfIndent(src, lineStart) - if (offset <= lineStart + indent) { - char = src[offset] - if ( - offset < lineStart + indent || - (char !== ']' && char !== '}') - ) { - const msg = 'Insufficient indentation in flow collection' - this.error = new YAMLSemanticError(this, msg) - } - } - } - break - case ',': - { - this.items.push({ char, offset }) - offset += 1 - } - break - case '#': - { - const comment = new Comment() - offset = comment.parse({ src }, offset) - this.items.push(comment) - } - break - case '?': - case ':': { - const next = src[offset + 1] - if ( - next === '\n' || - next === '\t' || - next === ' ' || - next === ',' || - // in-flow : after JSON-like key does not need to be followed by whitespace - (char === ':' && this.prevNodeIsJsonLike()) - ) { - this.items.push({ char, offset }) - offset += 1 - break - } - } - // fallthrough - default: { - const node = parseNode( - { - atLineStart: false, - inCollection: false, - inFlow: true, - indent: -1, - lineStart, - parent: this - }, - offset - ) - if (!node) { - // at next document start - this.valueRange = new Range(start, offset) - return offset - } - this.items.push(node) - offset = Node.normalizeOffset(src, node.range.end) - } - } - offset = Node.endOfWhiteSpace(src, offset) - char = src[offset] - } - this.valueRange = new Range(start, offset + 1) - if (char) { - this.items.push({ char, offset }) - offset = Node.endOfWhiteSpace(src, offset + 1) - offset = this.parseComment(offset) - } - trace: 'items', this.items, JSON.stringify(this.comment) - return offset - } - - setOrigRanges(cr, offset) { - offset = super.setOrigRanges(cr, offset) - this.items.forEach(node => { - if (node instanceof Node) { - offset = node.setOrigRanges(cr, offset) - } else if (cr.length === 0) { - node.origOffset = node.offset - } else { - let i = offset - while (i < cr.length) { - if (cr[i] > node.offset) break - else ++i - } - node.origOffset = node.offset + i - offset = i - } - }) - return offset - } - - toString() { - const { - context: { src }, - items, - range, - value - } = this - if (value != null) return value - const nodes = items.filter(item => item instanceof Node) - let str = '' - let prevEnd = range.start - nodes.forEach(node => { - const prefix = src.slice(prevEnd, node.range.start) - prevEnd = node.range.end - str += prefix + String(node) - if ( - str[str.length - 1] === '\n' && - src[prevEnd - 1] !== '\n' && - src[prevEnd] === '\n' - ) { - // Comment range does not include the terminal newline, but its - // stringified value does. Without this fix, newlines at comment ends - // get duplicated. - prevEnd += 1 - } - }) - str += src.slice(prevEnd, range.end) - return Node.addStringTerminator(src, range.end, str) - } -} diff --git a/src/cst/Node.js b/src/cst/Node.js deleted file mode 100644 index e89bccdd..00000000 --- a/src/cst/Node.js +++ /dev/null @@ -1,300 +0,0 @@ -import { Char, Type } from '../constants.js' -import { getLinePos } from './source-utils.js' -import { Range } from './Range.js' - -/** Root class of all nodes */ -export class Node { - static addStringTerminator(src, offset, str) { - if (str[str.length - 1] === '\n') return str - const next = Node.endOfWhiteSpace(src, offset) - return next >= src.length || src[next] === '\n' ? str + '\n' : str - } - - // ^(---|...) - static atDocumentBoundary(src, offset, sep) { - const ch0 = src[offset] - if (!ch0) return true - const prev = src[offset - 1] - if (prev && prev !== '\n') return false - if (sep) { - if (ch0 !== sep) return false - } else { - if (ch0 !== Char.DIRECTIVES_END && ch0 !== Char.DOCUMENT_END) return false - } - const ch1 = src[offset + 1] - const ch2 = src[offset + 2] - if (ch1 !== ch0 || ch2 !== ch0) return false - const ch3 = src[offset + 3] - return !ch3 || ch3 === '\n' || ch3 === '\t' || ch3 === ' ' - } - - static endOfIdentifier(src, offset) { - let ch = src[offset] - const isVerbatim = ch === '<' - const notOk = isVerbatim - ? ['\n', '\t', ' ', '>'] - : ['\n', '\t', ' ', '[', ']', '{', '}', ','] - while (ch && notOk.indexOf(ch) === -1) ch = src[(offset += 1)] - if (isVerbatim && ch === '>') offset += 1 - return offset - } - - static endOfIndent(src, offset) { - let ch = src[offset] - while (ch === ' ') ch = src[(offset += 1)] - return offset - } - - static endOfLine(src, offset) { - let ch = src[offset] - while (ch && ch !== '\n') ch = src[(offset += 1)] - return offset - } - - static endOfWhiteSpace(src, offset) { - let ch = src[offset] - while (ch === '\t' || ch === ' ') ch = src[(offset += 1)] - return offset - } - - static startOfLine(src, offset) { - let ch = src[offset - 1] - if (ch === '\n') return offset - while (ch && ch !== '\n') ch = src[(offset -= 1)] - return offset + 1 - } - - /** - * End of indentation, or null if the line's indent level is not more - * than `indent` - * - * @param {string} src - * @param {number} indent - * @param {number} lineStart - * @returns {?number} - */ - static endOfBlockIndent(src, indent, lineStart) { - const inEnd = Node.endOfIndent(src, lineStart) - if (inEnd > lineStart + indent) { - return inEnd - } else { - const wsEnd = Node.endOfWhiteSpace(src, inEnd) - const ch = src[wsEnd] - if (!ch || ch === '\n') return wsEnd - } - return null - } - - static atBlank(src, offset, endAsBlank) { - const ch = src[offset] - return ch === '\n' || ch === '\t' || ch === ' ' || (endAsBlank && !ch) - } - - static nextNodeIsIndented(ch, indentDiff, indicatorAsIndent) { - if (!ch || indentDiff < 0) return false - if (indentDiff > 0) return true - return indicatorAsIndent && ch === '-' - } - - // should be at line or string end, or at next non-whitespace char - static normalizeOffset(src, offset) { - const ch = src[offset] - return !ch - ? offset - : ch !== '\n' && src[offset - 1] === '\n' - ? offset - 1 - : Node.endOfWhiteSpace(src, offset) - } - - // fold single newline into space, multiple newlines to N - 1 newlines - // presumes src[offset] === '\n' - static foldNewline(src, offset, indent) { - let inCount = 0 - let error = false - let fold = '' - let ch = src[offset + 1] - while (ch === ' ' || ch === '\t' || ch === '\n') { - switch (ch) { - case '\n': - inCount = 0 - offset += 1 - fold += '\n' - break - case '\t': - if (inCount <= indent) error = true - offset = Node.endOfWhiteSpace(src, offset + 2) - 1 - break - case ' ': - inCount += 1 - offset += 1 - break - } - ch = src[offset + 1] - } - if (!fold) fold = ' ' - if (ch && inCount <= indent) error = true - return { fold, offset, error } - } - - constructor(type, props, context) { - Object.defineProperty(this, 'context', { - value: context || null, - writable: true - }) - this.error = null - this.range = null - this.valueRange = null - this.props = props || [] - this.type = type - this.value = null - } - - getPropValue(idx, key, skipKey) { - if (!this.context) return null - const { src } = this.context - const prop = this.props[idx] - return prop && src[prop.start] === key - ? src.slice(prop.start + (skipKey ? 1 : 0), prop.end) - : null - } - - get anchor() { - for (let i = 0; i < this.props.length; ++i) { - const anchor = this.getPropValue(i, Char.ANCHOR, true) - if (anchor != null) return anchor - } - return null - } - - get comment() { - const comments = [] - for (let i = 0; i < this.props.length; ++i) { - const comment = this.getPropValue(i, Char.COMMENT, true) - if (comment != null) comments.push(comment) - } - return comments.length > 0 ? comments.join('\n') : null - } - - commentHasRequiredWhitespace(start) { - const { src } = this.context - if (this.header && start === this.header.end) return false - if (!this.valueRange) return false - const { end } = this.valueRange - return start !== end || Node.atBlank(src, end - 1) - } - - get hasComment() { - if (this.context) { - const { src } = this.context - for (let i = 0; i < this.props.length; ++i) { - if (src[this.props[i].start] === Char.COMMENT) return true - } - } - return false - } - - get hasProps() { - if (this.context) { - const { src } = this.context - for (let i = 0; i < this.props.length; ++i) { - if (src[this.props[i].start] !== Char.COMMENT) return true - } - } - return false - } - - get includesTrailingLines() { - return false - } - - get jsonLike() { - const jsonLikeTypes = [ - Type.FLOW_MAP, - Type.FLOW_SEQ, - Type.QUOTE_DOUBLE, - Type.QUOTE_SINGLE - ] - return jsonLikeTypes.indexOf(this.type) !== -1 - } - - get rangeAsLinePos() { - if (!this.range || !this.context) return undefined - const start = getLinePos(this.range.start, this.context.root) - if (!start) return undefined - const end = getLinePos(this.range.end, this.context.root) - return { start, end } - } - - get rawValue() { - if (!this.valueRange || !this.context) return null - const { start, end } = this.valueRange - return this.context.src.slice(start, end) - } - - get tag() { - for (let i = 0; i < this.props.length; ++i) { - const tag = this.getPropValue(i, Char.TAG, false) - if (tag != null) { - if (tag[1] === '<') { - return { verbatim: tag.slice(2, -1) } - } else { - // eslint-disable-next-line no-unused-vars - const [_, handle, suffix] = tag.match(/^(.*!)([^!]*)$/) - return { handle, suffix } - } - } - } - return null - } - - get valueRangeContainsNewline() { - if (!this.valueRange || !this.context) return false - const { start, end } = this.valueRange - const { src } = this.context - for (let i = start; i < end; ++i) { - if (src[i] === '\n') return true - } - return false - } - - parseComment(start) { - const { src } = this.context - if (src[start] === Char.COMMENT) { - const end = Node.endOfLine(src, start + 1) - const commentRange = new Range(start, end) - this.props.push(commentRange) - trace: commentRange, - JSON.stringify( - this.getPropValue(this.props.length - 1, Char.COMMENT, true) - ) - return end - } - return start - } - - /** - * Populates the `origStart` and `origEnd` values of all ranges for this - * node. Extended by child classes to handle descendant nodes. - * - * @param {number[]} cr - Positions of dropped CR characters - * @param {number} offset - Starting index of `cr` from the last call - * @returns {number} - The next offset, matching the one found for `origStart` - */ - setOrigRanges(cr, offset) { - if (this.range) offset = this.range.setOrigRange(cr, offset) - if (this.valueRange) this.valueRange.setOrigRange(cr, offset) - this.props.forEach(prop => prop.setOrigRange(cr, offset)) - return offset - } - - toString() { - const { - context: { src }, - range, - value - } = this - if (value != null) return value - const str = src.slice(range.start, range.end) - return Node.addStringTerminator(src, range.end, str) - } -} diff --git a/src/cst/ParseContext.js b/src/cst/ParseContext.js deleted file mode 100644 index 7b716a8d..00000000 --- a/src/cst/ParseContext.js +++ /dev/null @@ -1,228 +0,0 @@ -import { Char, Type } from '../constants.js' -import { YAMLSyntaxError } from '../errors.js' -import { Alias } from './Alias.js' -import { BlockValue } from './BlockValue.js' -import { Collection } from './Collection.js' -import { CollectionItem } from './CollectionItem.js' -import { FlowCollection } from './FlowCollection.js' -import { Node } from './Node.js' -import { PlainValue } from './PlainValue.js' -import { QuoteDouble } from './QuoteDouble.js' -import { QuoteSingle } from './QuoteSingle.js' -import { Range } from './Range.js' - -function createNewNode(type, props) { - switch (type) { - case Type.ALIAS: - return new Alias(type, props) - case Type.BLOCK_FOLDED: - case Type.BLOCK_LITERAL: - return new BlockValue(type, props) - case Type.FLOW_MAP: - case Type.FLOW_SEQ: - return new FlowCollection(type, props) - case Type.MAP_KEY: - case Type.MAP_VALUE: - case Type.SEQ_ITEM: - return new CollectionItem(type, props) - case Type.COMMENT: - case Type.PLAIN: - return new PlainValue(type, props) - case Type.QUOTE_DOUBLE: - return new QuoteDouble(type, props) - case Type.QUOTE_SINGLE: - return new QuoteSingle(type, props) - /* istanbul ignore next */ - default: - return null // should never happen - } -} - -/** - * @param {boolean} atLineStart - Node starts at beginning of line - * @param {boolean} inFlow - true if currently in a flow context - * @param {boolean} inCollection - true if currently in a collection context - * @param {number} indent - Current level of indentation - * @param {number} lineStart - Start of the current line - * @param {Node} parent - The parent of the node - * @param {string} src - Source of the YAML document - */ -export class ParseContext { - static parseType(src, offset, inFlow) { - switch (src[offset]) { - case '*': - return Type.ALIAS - case '>': - return Type.BLOCK_FOLDED - case '|': - return Type.BLOCK_LITERAL - case '{': - return Type.FLOW_MAP - case '[': - return Type.FLOW_SEQ - case '?': - return !inFlow && Node.atBlank(src, offset + 1, true) - ? Type.MAP_KEY - : Type.PLAIN - case ':': - return !inFlow && Node.atBlank(src, offset + 1, true) - ? Type.MAP_VALUE - : Type.PLAIN - case '-': - return !inFlow && Node.atBlank(src, offset + 1, true) - ? Type.SEQ_ITEM - : Type.PLAIN - case '"': - return Type.QUOTE_DOUBLE - case "'": - return Type.QUOTE_SINGLE - default: - return Type.PLAIN - } - } - - constructor( - orig = {}, - { atLineStart, inCollection, inFlow, indent, lineStart, parent } = {} - ) { - this.atLineStart = - atLineStart != null ? atLineStart : orig.atLineStart || false - this.inCollection = - inCollection != null ? inCollection : orig.inCollection || false - this.inFlow = inFlow != null ? inFlow : orig.inFlow || false - this.indent = indent != null ? indent : orig.indent - this.lineStart = lineStart != null ? lineStart : orig.lineStart - this.parent = parent != null ? parent : orig.parent || {} - this.root = orig.root - this.src = orig.src - } - - nodeStartsCollection(node) { - const { inCollection, inFlow, src } = this - if (inCollection || inFlow) return false - if (node instanceof CollectionItem) return true - // check for implicit key - let offset = node.range.end - if (src[offset] === '\n' || src[offset - 1] === '\n') return false - offset = Node.endOfWhiteSpace(src, offset) - return src[offset] === ':' - } - - // Anchor and tag are before type, which determines the node implementation - // class; hence this intermediate step. - parseProps(offset) { - const { inFlow, parent, src } = this - const props = [] - let lineHasProps = false - offset = this.atLineStart - ? Node.endOfIndent(src, offset) - : Node.endOfWhiteSpace(src, offset) - let ch = src[offset] - while ( - ch === Char.ANCHOR || - ch === Char.COMMENT || - ch === Char.TAG || - ch === '\n' - ) { - if (ch === '\n') { - const lineStart = offset + 1 - const inEnd = Node.endOfIndent(src, lineStart) - const indentDiff = inEnd - (lineStart + this.indent) - const noIndicatorAsIndent = - parent.type === Type.SEQ_ITEM && parent.context.atLineStart - if ( - !Node.nextNodeIsIndented(src[inEnd], indentDiff, !noIndicatorAsIndent) - ) - break - this.atLineStart = true - this.lineStart = lineStart - lineHasProps = false - offset = inEnd - } else if (ch === Char.COMMENT) { - const end = Node.endOfLine(src, offset + 1) - props.push(new Range(offset, end)) - offset = end - } else { - let end = Node.endOfIdentifier(src, offset + 1) - if ( - ch === Char.TAG && - src[end] === ',' && - /^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+,\d\d\d\d(-\d\d){0,2}\/\S/.test( - src.slice(offset + 1, end + 13) - ) - ) { - // Let's presume we're dealing with a YAML 1.0 domain tag here, rather - // than an empty but 'foo.bar' private-tagged node in a flow collection - // followed without whitespace by a plain string starting with a year - // or date divided by something. - end = Node.endOfIdentifier(src, end + 5) - } - props.push(new Range(offset, end)) - lineHasProps = true - offset = Node.endOfWhiteSpace(src, end) - } - ch = src[offset] - } - // '- &a : b' has an anchor on an empty node - if (lineHasProps && ch === ':' && Node.atBlank(src, offset + 1, true)) - offset -= 1 - const type = ParseContext.parseType(src, offset, inFlow) - trace: 'props', type, { props, offset } - return { props, type, valueStart: offset } - } - - /** - * Parses a node from the source - * @param {ParseContext} overlay - * @param {number} start - Index of first non-whitespace character for the node - * @returns {?Node} - null if at a document boundary - */ - parseNode = (overlay, start) => { - if (Node.atDocumentBoundary(this.src, start)) return null - const context = new ParseContext(this, overlay) - const { props, type, valueStart } = context.parseProps(start) - trace: 'START', - { start, valueStart, type, props }, - { - start: `${context.lineStart} + ${context.indent}`, - atLineStart: context.atLineStart, - inCollection: context.inCollection, - inFlow: context.inFlow, - parent: context.parent.type - } - const node = createNewNode(type, props) - let offset = node.parse(context, valueStart) - node.range = new Range(start, offset) - /* istanbul ignore if */ - if (offset <= start) { - // This should never happen, but if it does, let's make sure to at least - // step one character forward to avoid a busy loop. - node.error = new Error(`Node#parse consumed no characters`) - node.error.parseEnd = offset - node.error.source = node - node.range.end = start + 1 - } - trace: node.type, node.range, JSON.stringify(node.rawValue) - if (context.nodeStartsCollection(node)) { - trace: 'collection-start' - if ( - !node.error && - !context.atLineStart && - context.parent.type === Type.DOCUMENT - ) { - node.error = new YAMLSyntaxError( - node, - 'Block collection must not have preceding content here (e.g. directives-end indicator)' - ) - } - const collection = new Collection(node) - offset = collection.parse(new ParseContext(context), offset) - collection.range = new Range(start, offset) - trace: collection.type, - collection.range, - JSON.stringify(collection.rawValue) - return collection - } - return node - } -} diff --git a/src/cst/PlainValue.js b/src/cst/PlainValue.js deleted file mode 100644 index 493cf67b..00000000 --- a/src/cst/PlainValue.js +++ /dev/null @@ -1,146 +0,0 @@ -import { YAMLSemanticError } from '../errors.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class PlainValue extends Node { - static endOfLine(src, start, inFlow) { - let ch = src[start] - let offset = start - while (ch && ch !== '\n') { - if ( - inFlow && - (ch === '[' || ch === ']' || ch === '{' || ch === '}' || ch === ',') - ) - break - const next = src[offset + 1] - if ( - ch === ':' && - (!next || - next === '\n' || - next === '\t' || - next === ' ' || - (inFlow && next === ',')) - ) - break - if ((ch === ' ' || ch === '\t') && next === '#') break - offset += 1 - ch = next - } - return offset - } - - get strValue() { - if (!this.valueRange || !this.context) return null - let { start, end } = this.valueRange - const { src } = this.context - let ch = src[end - 1] - while (start < end && (ch === '\n' || ch === '\t' || ch === ' ')) - ch = src[--end - 1] - let str = '' - for (let i = start; i < end; ++i) { - const ch = src[i] - if (ch === '\n') { - const { fold, offset } = Node.foldNewline(src, i, -1) - str += fold - i = offset - } else if (ch === ' ' || ch === '\t') { - // trim trailing whitespace - const wsStart = i - let next = src[i + 1] - while (i < end && (next === ' ' || next === '\t')) { - i += 1 - next = src[i + 1] - } - if (next !== '\n') str += i > wsStart ? src.slice(wsStart, i + 1) : ch - } else { - str += ch - } - } - const ch0 = src[start] - switch (ch0) { - case '\t': { - const msg = 'Plain value cannot start with a tab character' - const errors = [new YAMLSemanticError(this, msg)] - return { errors, str } - } - case '@': - case '`': { - const msg = `Plain value cannot start with reserved character ${ch0}` - const errors = [new YAMLSemanticError(this, msg)] - return { errors, str } - } - default: - return str - } - } - - parseBlockValue(start) { - const { indent, inFlow, src } = this.context - let offset = start - let valueEnd = start - for (let ch = src[offset]; ch === '\n'; ch = src[offset]) { - if (Node.atDocumentBoundary(src, offset + 1)) break - const end = Node.endOfBlockIndent(src, indent, offset + 1) - if (end === null || src[end] === '#') break - if (src[end] === '\n') { - offset = end - } else { - valueEnd = PlainValue.endOfLine(src, end, inFlow) - offset = valueEnd - } - } - if (this.valueRange.isEmpty()) this.valueRange.start = start - this.valueRange.end = valueEnd - trace: this.valueRange, JSON.stringify(this.rawValue) - return valueEnd - } - - /** - * Parses a plain value from the source - * - * Accepted forms are: - * ``` - * #comment - * - * first line - * - * first line #comment - * - * first line - * block - * lines - * - * #comment - * block - * lines - * ``` - * where block lines are empty or have an indent level greater than `indent`. - * - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this scalar, may be `\n` - */ - parse(context, start) { - this.context = context - trace: 'plain-start', context.pretty, { start } - const { inFlow, src } = context - let offset = start - const ch = src[offset] - if (ch && ch !== '#' && ch !== '\n') { - offset = PlainValue.endOfLine(src, start, inFlow) - } - this.valueRange = new Range(start, offset) - offset = Node.endOfWhiteSpace(src, offset) - offset = this.parseComment(offset) - trace: 'first line', - { offset, valueRange: this.valueRange, comment: this.comment }, - JSON.stringify(this.rawValue) - if (!this.hasComment || this.valueRange.isEmpty()) { - offset = this.parseBlockValue(offset) - } - trace: this.type, - { offset, valueRange: this.valueRange }, - JSON.stringify(this.rawValue) - return offset - } -} diff --git a/src/cst/QuoteDouble.js b/src/cst/QuoteDouble.js deleted file mode 100644 index 394c268d..00000000 --- a/src/cst/QuoteDouble.js +++ /dev/null @@ -1,182 +0,0 @@ -import { YAMLSemanticError, YAMLSyntaxError } from '../errors.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class QuoteDouble extends Node { - static endOfQuote(src, offset) { - let ch = src[offset] - while (ch && ch !== '"') { - offset += ch === '\\' ? 2 : 1 - ch = src[offset] - } - return offset + 1 - } - - /** - * @returns {string | { str: string, errors: YAMLSyntaxError[] }} - */ - get strValue() { - if (!this.valueRange || !this.context) return null - const errors = [] - const { start, end } = this.valueRange - const { indent, src } = this.context - if (src[end - 1] !== '"') - errors.push(new YAMLSyntaxError(this, 'Missing closing "quote')) - // Using String#replace is too painful with escaped newlines preceded by - // escaped backslashes; also, this should be faster. - let str = '' - for (let i = start + 1; i < end - 1; ++i) { - const ch = src[i] - if (ch === '\n') { - if (Node.atDocumentBoundary(src, i + 1)) - errors.push( - new YAMLSemanticError( - this, - 'Document boundary indicators are not allowed within string values' - ) - ) - const { fold, offset, error } = Node.foldNewline(src, i, indent) - str += fold - i = offset - if (error) - errors.push( - new YAMLSemanticError( - this, - 'Multi-line double-quoted string needs to be sufficiently indented' - ) - ) - } else if (ch === '\\') { - i += 1 - switch (src[i]) { - case '0': - str += '\0' - break // null character - case 'a': - str += '\x07' - break // bell character - case 'b': - str += '\b' - break // backspace - case 'e': - str += '\x1b' - break // escape character - case 'f': - str += '\f' - break // form feed - case 'n': - str += '\n' - break // line feed - case 'r': - str += '\r' - break // carriage return - case 't': - str += '\t' - break // horizontal tab - case 'v': - str += '\v' - break // vertical tab - case 'N': - str += '\u0085' - break // Unicode next line - case '_': - str += '\u00a0' - break // Unicode non-breaking space - case 'L': - str += '\u2028' - break // Unicode line separator - case 'P': - str += '\u2029' - break // Unicode paragraph separator - case ' ': - str += ' ' - break - case '"': - str += '"' - break - case '/': - str += '/' - break - case '\\': - str += '\\' - break - case '\t': - str += '\t' - break - case 'x': - str += this.parseCharCode(i + 1, 2, errors) - i += 2 - break - case 'u': - str += this.parseCharCode(i + 1, 4, errors) - i += 4 - break - case 'U': - str += this.parseCharCode(i + 1, 8, errors) - i += 8 - break - case '\n': - // skip escaped newlines, but still trim the following line - while (src[i + 1] === ' ' || src[i + 1] === '\t') i += 1 - break - default: - errors.push( - new YAMLSyntaxError( - this, - `Invalid escape sequence ${src.substr(i - 1, 2)}` - ) - ) - str += '\\' + src[i] - } - } else if (ch === ' ' || ch === '\t') { - // trim trailing whitespace - const wsStart = i - let next = src[i + 1] - while (next === ' ' || next === '\t') { - i += 1 - next = src[i + 1] - } - if (next !== '\n') str += i > wsStart ? src.slice(wsStart, i + 1) : ch - } else { - str += ch - } - } - return errors.length > 0 ? { errors, str } : str - } - - parseCharCode(offset, length, errors) { - const { src } = this.context - const cc = src.substr(offset, length) - const ok = cc.length === length && /^[0-9a-fA-F]+$/.test(cc) - const code = ok ? parseInt(cc, 16) : NaN - if (isNaN(code)) { - errors.push( - new YAMLSyntaxError( - this, - `Invalid escape sequence ${src.substr(offset - 2, length + 2)}` - ) - ) - return src.substr(offset - 2, length + 2) - } - return String.fromCodePoint(code) - } - - /** - * Parses a "double quoted" value from the source - * - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this scalar - */ - parse(context, start) { - this.context = context - const { src } = context - let offset = QuoteDouble.endOfQuote(src, start + 1) - this.valueRange = new Range(start, offset) - offset = Node.endOfWhiteSpace(src, offset) - offset = this.parseComment(offset) - trace: this.type, - { valueRange: this.valueRange, comment: this.comment }, - JSON.stringify(this.rawValue) - return offset - } -} diff --git a/src/cst/QuoteSingle.js b/src/cst/QuoteSingle.js deleted file mode 100644 index 9f9929fd..00000000 --- a/src/cst/QuoteSingle.js +++ /dev/null @@ -1,95 +0,0 @@ -import { YAMLSemanticError, YAMLSyntaxError } from '../errors.js' -import { Node } from './Node.js' -import { Range } from './Range.js' - -export class QuoteSingle extends Node { - static endOfQuote(src, offset) { - let ch = src[offset] - while (ch) { - if (ch === "'") { - if (src[offset + 1] !== "'") break - ch = src[(offset += 2)] - } else { - ch = src[(offset += 1)] - } - } - return offset + 1 - } - - /** - * @returns {string | { str: string, errors: YAMLSyntaxError[] }} - */ - get strValue() { - if (!this.valueRange || !this.context) return null - const errors = [] - const { start, end } = this.valueRange - const { indent, src } = this.context - if (src[end - 1] !== "'") - errors.push(new YAMLSyntaxError(this, "Missing closing 'quote")) - let str = '' - for (let i = start + 1; i < end - 1; ++i) { - const ch = src[i] - if (ch === '\n') { - if (Node.atDocumentBoundary(src, i + 1)) - errors.push( - new YAMLSemanticError( - this, - 'Document boundary indicators are not allowed within string values' - ) - ) - const { fold, offset, error } = Node.foldNewline(src, i, indent) - str += fold - i = offset - if (error) - errors.push( - new YAMLSemanticError( - this, - 'Multi-line single-quoted string needs to be sufficiently indented' - ) - ) - } else if (ch === "'") { - str += ch - i += 1 - if (src[i] !== "'") - errors.push( - new YAMLSyntaxError( - this, - 'Unescaped single quote? This should not happen.' - ) - ) - } else if (ch === ' ' || ch === '\t') { - // trim trailing whitespace - const wsStart = i - let next = src[i + 1] - while (next === ' ' || next === '\t') { - i += 1 - next = src[i + 1] - } - if (next !== '\n') str += i > wsStart ? src.slice(wsStart, i + 1) : ch - } else { - str += ch - } - } - return errors.length > 0 ? { errors, str } : str - } - - /** - * Parses a 'single quoted' value from the source - * - * @param {ParseContext} context - * @param {number} start - Index of first character - * @returns {number} - Index of the character after this scalar - */ - parse(context, start) { - this.context = context - const { src } = context - let offset = QuoteSingle.endOfQuote(src, start + 1) - this.valueRange = new Range(start, offset) - offset = Node.endOfWhiteSpace(src, offset) - offset = this.parseComment(offset) - trace: this.type, - { valueRange: this.valueRange, comment: this.comment }, - JSON.stringify(this.rawValue) - return offset - } -} diff --git a/src/cst/README.md b/src/cst/README.md deleted file mode 100644 index 5bb5a1da..00000000 --- a/src/cst/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# YAML CST Parser - -Documentation for the CST parser has been moved to diff --git a/src/cst/Range.js b/src/cst/Range.js deleted file mode 100644 index a24f54b1..00000000 --- a/src/cst/Range.js +++ /dev/null @@ -1,45 +0,0 @@ -export class Range { - static copy(orig) { - return new Range(orig.start, orig.end) - } - - constructor(start, end) { - this.start = start - this.end = end || start - } - - isEmpty() { - return typeof this.start !== 'number' || !this.end || this.end <= this.start - } - - /** - * Set `origStart` and `origEnd` to point to the original source range for - * this node, which may differ due to dropped CR characters. - * - * @param {number[]} cr - Positions of dropped CR characters - * @param {number} offset - Starting index of `cr` from the last call - * @returns {number} - The next offset, matching the one found for `origStart` - */ - setOrigRange(cr, offset) { - const { start, end } = this - if (cr.length === 0 || end <= cr[0]) { - this.origStart = start - this.origEnd = end - return offset - } - let i = offset - while (i < cr.length) { - if (cr[i] > start) break - else ++i - } - this.origStart = start + i - const nextOffset = i - while (i < cr.length) { - // if end was at \n, it should now be at \r - if (cr[i] >= end) break - else ++i - } - this.origEnd = end + i - return nextOffset - } -} diff --git a/src/cst/index.d.ts b/src/cst/index.d.ts deleted file mode 100644 index d67d1f18..00000000 --- a/src/cst/index.d.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { Type } from '../constants' -import { YAMLSyntaxError } from '../errors' - -export namespace CST { - interface Range { - start: number - end: number - origStart?: number - origEnd?: number - isEmpty(): boolean - } - - interface ParseContext { - /** Node starts at beginning of line */ - atLineStart: boolean - /** true if currently in a collection context */ - inCollection: boolean - /** true if currently in a flow context */ - inFlow: boolean - /** Current level of indentation */ - indent: number - /** Start of the current line */ - lineStart: number - /** The parent of the node */ - parent: Node - /** Source of the YAML document */ - src: string - } - - interface Node { - context: ParseContext | null - /** if not null, indicates a parser failure */ - error: YAMLSyntaxError | null - /** span of context.src parsed into this node */ - range: Range | null - valueRange: Range | null - /** anchors, tags and comments */ - props: Range[] - /** specific node type */ - type: Type - /** if non-null, overrides source value */ - value: string | null - - readonly anchor: string | null - readonly comment: string | null - readonly hasComment: boolean - readonly hasProps: boolean - readonly jsonLike: boolean - readonly rawValue: string | null - readonly tag: - | null - | { verbatim: string } - | { handle: string; suffix: string } - readonly valueRangeContainsNewline: boolean - } - - interface Alias extends Node { - type: Type.ALIAS - /** contain the anchor without the * prefix */ - readonly rawValue: string - } - - type Scalar = BlockValue | PlainValue | QuoteValue - - interface BlockValue extends Node { - type: Type.BLOCK_FOLDED | Type.BLOCK_LITERAL - chomping: 'CLIP' | 'KEEP' | 'STRIP' - blockIndent: number | null - header: Range - readonly strValue: string | null - } - - interface BlockFolded extends BlockValue { - type: Type.BLOCK_FOLDED - } - - interface BlockLiteral extends BlockValue { - type: Type.BLOCK_LITERAL - } - - interface PlainValue extends Node { - type: Type.PLAIN - readonly strValue: string | null - } - - interface QuoteValue extends Node { - type: Type.QUOTE_DOUBLE | Type.QUOTE_SINGLE - readonly strValue: - | null - | string - | { str: string; errors: YAMLSyntaxError[] } - } - - interface QuoteDouble extends QuoteValue { - type: Type.QUOTE_DOUBLE - } - - interface QuoteSingle extends QuoteValue { - type: Type.QUOTE_SINGLE - } - - interface Comment extends Node { - type: Type.COMMENT - readonly anchor: null - readonly comment: string - readonly rawValue: null - readonly tag: null - } - - interface BlankLine extends Node { - type: Type.BLANK_LINE - } - - interface MapItem extends Node { - type: Type.MAP_KEY | Type.MAP_VALUE - node: ContentNode | null - } - - interface MapKey extends MapItem { - type: Type.MAP_KEY - } - - interface MapValue extends MapItem { - type: Type.MAP_VALUE - } - - interface Map extends Node { - type: Type.MAP - /** implicit keys are not wrapped */ - items: Array - } - - interface SeqItem extends Node { - type: Type.SEQ_ITEM - node: ContentNode | null - } - - interface Seq extends Node { - type: Type.SEQ - items: Array - } - - interface FlowChar { - char: '{' | '}' | '[' | ']' | ',' | '?' | ':' - offset: number - origOffset?: number - } - - interface FlowCollection extends Node { - type: Type.FLOW_MAP | Type.FLOW_SEQ - items: Array< - FlowChar | BlankLine | Comment | Alias | Scalar | FlowCollection - > - } - - interface FlowMap extends FlowCollection { - type: Type.FLOW_MAP - } - - interface FlowSeq extends FlowCollection { - type: Type.FLOW_SEQ - } - - type ContentNode = Alias | Scalar | Map | Seq | FlowCollection - - interface Directive extends Node { - type: Type.DIRECTIVE - name: string - readonly anchor: null - readonly parameters: string[] - readonly tag: null - } - - interface Document extends Node { - type: Type.DOCUMENT - directives: Array - contents: Array - readonly anchor: null - readonly comment: null - readonly tag: null - } -} diff --git a/src/cst/parse.js b/src/cst/parse.js deleted file mode 100644 index 613cc2a5..00000000 --- a/src/cst/parse.js +++ /dev/null @@ -1,32 +0,0 @@ -import { Document } from './Document.js' -import { ParseContext } from './ParseContext.js' - -export function parse(src) { - const cr = [] - if (src.indexOf('\r') !== -1) { - src = src.replace(/\r\n?/g, (match, offset) => { - if (match.length > 1) cr.push(offset) - return '\n' - }) - } - const documents = [] - let offset = 0 - do { - const doc = new Document() - const context = new ParseContext({ src }) - offset = doc.parse(context, offset) - documents.push(doc) - } while (offset < src.length) - documents.setOrigRanges = () => { - if (cr.length === 0) return false - for (let i = 1; i < cr.length; ++i) cr[i] -= i - let crOffset = 0 - for (let i = 0; i < documents.length; ++i) { - crOffset = documents[i].setOrigRanges(cr, crOffset) - } - cr.splice(0, cr.length) - return true - } - documents.toString = () => documents.join('...\n') - return documents -} diff --git a/src/cst/source-utils.js b/src/cst/source-utils.js deleted file mode 100644 index fc472644..00000000 --- a/src/cst/source-utils.js +++ /dev/null @@ -1,132 +0,0 @@ -function findLineStarts(src) { - const ls = [0] - let offset = src.indexOf('\n') - while (offset !== -1) { - offset += 1 - ls.push(offset) - offset = src.indexOf('\n', offset) - } - return ls -} - -function getSrcInfo(cst) { - let lineStarts, src - if (typeof cst === 'string') { - lineStarts = findLineStarts(cst) - src = cst - } else { - if (Array.isArray(cst)) cst = cst[0] - if (cst && cst.context) { - if (!cst.lineStarts) cst.lineStarts = findLineStarts(cst.context.src) - lineStarts = cst.lineStarts - src = cst.context.src - } - } - return { lineStarts, src } -} - -/** - * @typedef {Object} LinePos - One-indexed position in the source - * @property {number} line - * @property {number} col - */ - -/** - * Determine the line/col position matching a character offset. - * - * Accepts a source string or a CST document as the second parameter. With - * the latter, starting indices for lines are cached in the document as - * `lineStarts: number[]`. - * - * Returns a one-indexed `{ line, col }` location if found, or - * `undefined` otherwise. - * - * @param {number} offset - * @param {string|Document|Document[]} cst - * @returns {?LinePos} - */ -export function getLinePos(offset, cst) { - if (typeof offset !== 'number' || offset < 0) return null - const { lineStarts, src } = getSrcInfo(cst) - if (!lineStarts || !src || offset > src.length) return null - for (let i = 0; i < lineStarts.length; ++i) { - const start = lineStarts[i] - if (offset < start) { - return { line: i, col: offset - lineStarts[i - 1] + 1 } - } - if (offset === start) return { line: i + 1, col: 1 } - } - const line = lineStarts.length - return { line, col: offset - lineStarts[line - 1] + 1 } -} - -/** - * Get a specified line from the source. - * - * Accepts a source string or a CST document as the second parameter. With - * the latter, starting indices for lines are cached in the document as - * `lineStarts: number[]`. - * - * Returns the line as a string if found, or `null` otherwise. - * - * @param {number} line One-indexed line number - * @param {string|Document|Document[]} cst - * @returns {?string} - */ -export function getLine(line, cst) { - const { lineStarts, src } = getSrcInfo(cst) - if (!lineStarts || !(line >= 1) || line > lineStarts.length) return null - const start = lineStarts[line - 1] - let end = lineStarts[line] // undefined for last line; that's ok for slice() - while (end && end > start && src[end - 1] === '\n') --end - return src.slice(start, end) -} - -/** - * Pretty-print the starting line from the source indicated by the range `pos` - * - * Trims output to `maxWidth` chars while keeping the starting column visible, - * using `…` at either end to indicate dropped characters. - * - * Returns a two-line string (or `null`) with `\n` as separator; the second line - * will hold appropriately indented `^` marks indicating the column range. - * - * @param {Object} pos - * @param {LinePos} pos.start - * @param {LinePos} [pos.end] - * @param {string|Document|Document[]*} cst - * @param {number} [maxWidth=80] - * @returns {?string} - */ -export function getPrettyContext({ start, end }, cst, maxWidth = 80) { - let src = getLine(start.line, cst) - if (!src) return null - let { col } = start - if (src.length > maxWidth) { - if (col <= maxWidth - 10) { - src = src.substr(0, maxWidth - 1) + '…' - } else { - const halfWidth = Math.round(maxWidth / 2) - if (src.length > col + halfWidth) - src = src.substr(0, col + halfWidth - 1) + '…' - col -= src.length - maxWidth - src = '…' + src.substr(1 - maxWidth) - } - } - let errLen = 1 - let errEnd = '' - if (end) { - if ( - end.line === start.line && - col + (end.col - start.col) <= maxWidth + 1 - ) { - errLen = end.col - start.col - } else { - errLen = Math.min(src.length + 1, maxWidth) - col - errEnd = '…' - } - } - const offset = col > 1 ? ' '.repeat(col - 1) : '' - const err = '^'.repeat(errLen) - return `${src}\n${offset}${err}${errEnd}` -} diff --git a/src/doc/Document.d.ts b/src/doc/Document.d.ts index 864d8051..e7580e56 100644 --- a/src/doc/Document.d.ts +++ b/src/doc/Document.d.ts @@ -1,6 +1,5 @@ import { Alias, Collection, Merge, Node, Pair } from '../ast' import { Type } from '../constants' -import { CST } from '../cst' import { YAMLError, YAMLWarning } from '../errors' import { Options } from '../options' import { Directives } from './directives' @@ -30,7 +29,7 @@ export interface CreateNodeOptions { } export class Document extends Collection { - cstNode?: CST.Document + // cstNode?: CST.Document /** * @param value - The initial value for the document, which will be wrapped * in a Node container. @@ -135,9 +134,7 @@ export namespace Document { interface Anchors { /** @private */ - _cstAliases: any[] - /** @private */ - map: Record + map: Record /** * Create a new `Alias` node, adding the required anchor for `node`. diff --git a/src/doc/Schema.d.ts b/src/doc/Schema.d.ts index d30a3e24..19503a41 100644 --- a/src/doc/Schema.d.ts +++ b/src/doc/Schema.d.ts @@ -1,6 +1,4 @@ -import { Document } from './Document' import { Node, Pair, Scalar, YAMLMap, YAMLSeq } from '../ast' -import { CST } from '../cst' export class Schema { constructor(options: Schema.Options) diff --git a/src/errors.d.ts b/src/errors.d.ts index 657e2394..1c820c43 100644 --- a/src/errors.d.ts +++ b/src/errors.d.ts @@ -1,5 +1,4 @@ -import { Type } from './constants' -import { CST } from './cst' +// import { Type } from './constants' interface LinePos { line: number @@ -15,11 +14,10 @@ export class YAMLError extends Error { | 'YAMLWarning' message: string offset?: number - source?: CST.Node - nodeType?: Type - range?: CST.Range - linePos?: { start: LinePos; end: LinePos } + // nodeType?: Type + // range?: CST.Range + // linePos?: { start: LinePos; end: LinePos } /** * Drops `source` and adds `nodeType`, `range` and `linePos`, as well as @@ -31,25 +29,25 @@ export class YAMLError extends Error { export class YAMLParseError extends YAMLError { name: 'YAMLParseError' - constructor(source: Node | number, message: string) + constructor(source: number, message: string) } export class YAMLReferenceError extends YAMLError { name: 'YAMLReferenceError' - constructor(source: Node | number, message: string) + constructor(source: number, message: string) } export class YAMLSemanticError extends YAMLError { name: 'YAMLSemanticError' - constructor(source: Node | number, message: string) + constructor(source: number, message: string) } export class YAMLSyntaxError extends YAMLError { name: 'YAMLSyntaxError' - constructor(source: Node | number, message: string) + constructor(source: number, message: string) } export class YAMLWarning extends YAMLError { name: 'YAMLWarning' - constructor(source: Node | number, message: string) + constructor(source: number, message: string) } diff --git a/src/errors.js b/src/errors.js index 0f4f3610..2fb44ecc 100644 --- a/src/errors.js +++ b/src/errors.js @@ -1,39 +1,34 @@ -import { Node } from './cst/Node.js' -import { getLinePos, getPrettyContext } from './cst/source-utils.js' -import { Range } from './cst/Range.js' - export class YAMLError extends Error { - constructor(name, source, message) { + constructor(name, offset, message) { if (!message) throw new Error(`Invalid arguments for new ${name}`) super() this.name = name this.message = message - if (source instanceof Node) this.source = source - else if (typeof source === 'number') this.offset = source + if (typeof offset === 'number') this.offset = offset } makePretty() { if (!this.source) return - this.nodeType = this.source.type - const cst = this.source.context && this.source.context.root - if (typeof this.offset === 'number') { - this.range = new Range(this.offset, this.offset + 1) - const start = cst && getLinePos(this.offset, cst) - if (start) { - const end = { line: start.line, col: start.col + 1 } - this.linePos = { start, end } - } - delete this.offset - } else { - this.range = this.source.range - this.linePos = this.source.rangeAsLinePos - } - if (this.linePos) { - const { line, col } = this.linePos.start - this.message += ` at line ${line}, column ${col}` - const ctx = cst && getPrettyContext(this.linePos, cst) - if (ctx) this.message += `:\n\n${ctx}\n` - } + // this.nodeType = this.source.type + // const cst = this.source.context && this.source.context.root + // if (typeof this.offset === 'number') { + // this.range = new Range(this.offset, this.offset + 1) + // const start = cst && getLinePos(this.offset, cst) + // if (start) { + // const end = { line: start.line, col: start.col + 1 } + // this.linePos = { start, end } + // } + // delete this.offset + // } else { + // this.range = this.source.range + // this.linePos = this.source.rangeAsLinePos + // } + // if (this.linePos) { + // const { line, col } = this.linePos.start + // this.message += ` at line ${line}, column ${col}` + // const ctx = cst && getPrettyContext(this.linePos, cst) + // if (ctx) this.message += `:\n\n${ctx}\n` + // } delete this.source } } diff --git a/tests/cst/Node.js b/tests/cst/Node.js deleted file mode 100644 index a3a6ce71..00000000 --- a/tests/cst/Node.js +++ /dev/null @@ -1,32 +0,0 @@ -import { Type } from '../../src/constants.js' -import { Node } from '../../src/cst/Node.js' -import { Range } from '../../src/cst/Range.js' - -describe('internals', () => { - test('constructor', () => { - const src = '&anchor !tag src' - const props = [new Range(0, 7), new Range(8, 12)] - const node = new Node(Type.PLAIN, props, { src }) - expect(node.context.src).toBe(src) - expect(node.anchor).toBe('anchor') - expect(node.tag).toMatchObject({ handle: '!', suffix: 'tag' }) - expect(node.type).toBe(Type.PLAIN) - }) - - test('.endOfIndent', () => { - const src = ' - value\n- other\n' - const in1 = Node.endOfIndent(src, 0) - expect(in1).toBe(2) - const offset = src.indexOf('\n') + 1 - const in2 = Node.endOfIndent(src, offset) - expect(in2).toBe(offset) - }) - - test('#parseComment', () => { - const src = '#comment here\nnext' - const node = new Node(Type.COMMENT, null, { src }) - const end = node.parseComment(0) - expect(node.comment).toBe('comment here') - expect(end).toBe(src.indexOf('\n')) - }) -}) diff --git a/tests/cst/YAML-1.2.spec.js b/tests/cst/YAML-1.2.spec.js deleted file mode 100644 index 901081e5..00000000 --- a/tests/cst/YAML-1.2.spec.js +++ /dev/null @@ -1,3608 +0,0 @@ -import { Type } from '../../src/constants.js' -import { parse } from '../../src/cst/parse.js' -import { pretty, testSpec } from './common.js' - -const spec = { - '2.1. Collections': { - 'Example 2.1. Sequence of Scalars': { - src: `- Mark McGwire\r -- Sammy Sosa\r -- Ken Griffey\r`, - tgt: [ - { - contents: [ - { - type: Type.SEQ, - items: [ - { type: Type.SEQ_ITEM, node: 'Mark McGwire' }, - { type: Type.SEQ_ITEM, node: 'Sammy Sosa' }, - { type: Type.SEQ_ITEM, node: 'Ken Griffey' } - ] - } - ] - } - ] - }, - - 'Example 2.2. Mapping Scalars to Scalars': { - src: `hr: 65 # Home runs\r -avg: 0.278 # Batting average\r -rbi: 147 # Runs Batted In`, - tgt: [ - { - contents: [ - { - type: Type.MAP, - items: [ - 'hr', - { - type: Type.MAP_VALUE, - node: { comment: ' Home runs', strValue: '65' } - }, - 'avg', - { - type: Type.MAP_VALUE, - node: { comment: ' Batting average', strValue: '0.278' } - }, - 'rbi', - { - type: Type.MAP_VALUE, - node: { comment: ' Runs Batted In', strValue: '147' } - } - ] - } - ] - } - ] - }, - - 'Example 2.3. Mapping Scalars to Sequences': { - src: `american:\r - - Boston Red Sox\r - - Detroit Tigers\r - - New York Yankees\r -national:\r - - New York Mets\r - - Chicago Cubs\r - - Atlanta Braves`, - tgt: [ - { - contents: [ - { - items: [ - 'american', - { - type: Type.MAP_VALUE, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'Boston Red Sox' }, - { type: Type.SEQ_ITEM, node: 'Detroit Tigers' }, - { type: Type.SEQ_ITEM, node: 'New York Yankees' } - ] - } - }, - 'national', - { - type: Type.MAP_VALUE, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'New York Mets' }, - { type: Type.SEQ_ITEM, node: 'Chicago Cubs' }, - { type: Type.SEQ_ITEM, node: 'Atlanta Braves' } - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 2.4. Sequence of Mappings': { - src: `- - name: Mark McGwire - hr: 65 - avg: 0.278 -- - name: Sammy Sosa - hr: 63 - avg: 0.288`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'name', - { type: Type.MAP_VALUE, node: 'Mark McGwire' }, - 'hr', - { type: Type.MAP_VALUE, node: '65' }, - 'avg', - { type: Type.MAP_VALUE, node: '0.278' } - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'name', - { type: Type.MAP_VALUE, node: 'Sammy Sosa' }, - 'hr', - { type: Type.MAP_VALUE, node: '63' }, - 'avg', - { type: Type.MAP_VALUE, node: '0.288' } - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 2.5. Sequence of Sequences': { - src: `- [name , hr, avg ] -- [Mark McGwire, 65, 0.278] -- [Sammy Sosa , 63, 0.288]`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { items: ['[', 'name', ',', 'hr', ',', 'avg', ']'] } - }, - { - type: Type.SEQ_ITEM, - node: { - items: ['[', 'Mark McGwire', ',', '65', ',', '0.278', ']'] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: ['[', 'Sammy Sosa', ',', '63', ',', '0.288', ']'] - } - } - ] - } - ] - } - ] - }, - - 'Example 2.6. Mapping of Mappings': { - src: `Mark McGwire: {hr: 65, avg: 0.278} -Sammy Sosa: { - hr: 63, - avg: 0.288 - }`, - tgt: [ - { - contents: [ - { - items: [ - 'Mark McGwire', - { - type: Type.MAP_VALUE, - node: { - items: ['{', 'hr', ':', '65', ',', 'avg', ':', '0.278', '}'] - } - }, - 'Sammy Sosa', - { - type: Type.MAP_VALUE, - node: { - items: ['{', 'hr', ':', '63', ',', 'avg', ':', '0.288', '}'] - } - } - ] - } - ] - } - ] - } - }, - - '2.2. Structures': { - 'Example 2.7. Two Documents in a Stream': { - src: `# Ranking of 1998 home runs ---- -- Mark McGwire -- Sammy Sosa -- Ken Griffey - -# Team ranking ---- -- Chicago Cubs -- St Louis Cardinals`, - tgt: [ - { - directives: [{ comment: ' Ranking of 1998 home runs' }], - contents: [ - { - items: [ - { type: Type.SEQ_ITEM, node: 'Mark McGwire' }, - { type: Type.SEQ_ITEM, node: 'Sammy Sosa' }, - { type: Type.SEQ_ITEM, node: 'Ken Griffey' } - ] - }, - { type: Type.BLANK_LINE }, - { comment: ' Team ranking' } - ] - }, - { - contents: [ - { - items: [ - { type: Type.SEQ_ITEM, node: 'Chicago Cubs' }, - { type: Type.SEQ_ITEM, node: 'St Louis Cardinals' } - ] - } - ] - } - ] - }, - - 'Example 2.8. Play by Play Feed': { - src: `--- -time: 20:03:20 -player: Sammy Sosa -action: strike (miss) -... ---- -time: 20:03:47 -player: Sammy Sosa -action: grand slam -...`, - tgt: [ - { - contents: [ - { - items: [ - 'time', - { type: Type.MAP_VALUE, node: '20:03:20' }, - 'player', - { type: Type.MAP_VALUE, node: 'Sammy Sosa' }, - 'action', - { type: Type.MAP_VALUE, node: 'strike (miss)' } - ] - } - ] - }, - { - contents: [ - { - items: [ - 'time', - { type: Type.MAP_VALUE, node: '20:03:47' }, - 'player', - { type: Type.MAP_VALUE, node: 'Sammy Sosa' }, - 'action', - { type: Type.MAP_VALUE, node: 'grand slam' } - ] - } - ] - } - ] - }, - - 'Example 2.9. Single Document with Two Comments': { - src: `--- -hr: # 1998 hr ranking - - Mark McGwire - - Sammy Sosa -rbi: - # 1998 rbi ranking - - Sammy Sosa - - Ken Griffey`, - tgt: [ - { - contents: [ - { - items: [ - 'hr', - { - comment: ' 1998 hr ranking', - type: Type.MAP_VALUE, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'Mark McGwire' }, - { type: Type.SEQ_ITEM, node: 'Sammy Sosa' } - ] - } - }, - 'rbi', - { - comment: ' 1998 rbi ranking', - type: Type.MAP_VALUE, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'Sammy Sosa' }, - { type: Type.SEQ_ITEM, node: 'Ken Griffey' } - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 2.10. Node for “Sammy Sosa” appears twice in this document': { - src: `--- -hr: - - Mark McGwire - # Following node labeled SS - - &SS Sammy Sosa -rbi: - - *SS # Subsequent occurrence - - Ken Griffey`, - tgt: [ - { - contents: [ - { - items: [ - 'hr', - { - type: Type.MAP_VALUE, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'Mark McGwire' }, - { comment: ' Following node labeled SS' }, - { - type: Type.SEQ_ITEM, - node: { anchor: 'SS', strValue: 'Sammy Sosa' } - } - ] - } - }, - 'rbi', - { - type: Type.MAP_VALUE, - node: { - items: [ - { - type: Type.SEQ_ITEM, - node: { - comment: ' Subsequent occurrence', - type: Type.ALIAS, - rawValue: 'SS' - } - }, - { type: Type.SEQ_ITEM, node: 'Ken Griffey' } - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 2.11. Mapping between Sequences': { - src: `? - Detroit Tigers - - Chicago cubs -: - - 2001-07-23 - -? [ New York Yankees, - Atlanta Braves ] -: [ 2001-07-02, 2001-08-12, - 2001-08-14 ]`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.MAP_KEY, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'Detroit Tigers' }, - { type: Type.SEQ_ITEM, node: 'Chicago cubs' } - ] - } - }, - { - type: Type.MAP_VALUE, - node: { - items: [{ type: Type.SEQ_ITEM, node: '2001-07-23' }] - } - }, - { type: Type.BLANK_LINE }, - { - type: Type.MAP_KEY, - node: { - items: ['[', 'New York Yankees', ',', 'Atlanta Braves', ']'] - } - }, - { - type: Type.MAP_VALUE, - node: { - items: [ - '[', - '2001-07-02', - ',', - '2001-08-12', - ',', - '2001-08-14', - ']' - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 2.12. Compact Nested Mapping': { - src: `--- -# Products purchased -- item : Super Hoop - quantity: 1 -- item : Basketball - quantity: 4 -- item : Big Shoes - quantity: 1`, - tgt: [ - { - contents: [ - { comment: ' Products purchased' }, - { - items: [ - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'item', - { type: Type.MAP_VALUE, node: 'Super Hoop' }, - 'quantity', - { type: Type.MAP_VALUE, node: '1' } - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'item', - { type: Type.MAP_VALUE, node: 'Basketball' }, - 'quantity', - { type: Type.MAP_VALUE, node: '4' } - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'item', - { type: Type.MAP_VALUE, node: 'Big Shoes' }, - 'quantity', - { type: Type.MAP_VALUE, node: '1' } - ] - } - } - ] - } - ] - } - ] - } - }, - - '2.3. Scalars': { - 'Example 2.13. In literals, newlines are preserved': { - src: `# ASCII Art ---- | - \\//||\\/|| - // || ||__`, - tgt: [ - { - directives: [{ comment: ' ASCII Art' }], - contents: ['\\//||\\/||\n// || ||__\n'] - } - ] - }, - - 'Example 2.14. In the folded scalars, newlines become spaces': { - src: `--- > - Mark McGwire's - year was crippled - by a knee injury.`, - tgt: [ - { contents: ["Mark McGwire's year was crippled by a knee injury.\n"] } - ] - }, - - 'Example 2.15. Folded newlines are preserved for "more indented" and blank lines': { - src: `> - Sammy Sosa completed another - fine season with great stats. - - 63 Home Runs - 0.288 Batting Average - - What a year!`, - tgt: [ - { - contents: [ - `Sammy Sosa completed another fine season with great stats. - - 63 Home Runs - 0.288 Batting Average - -What a year!\n` - ] - } - ] - }, - - 'Example 2.16. Indentation determines scope': { - src: `name: Mark McGwire -accomplishment: > - Mark set a major league - home run record in 1998. -stats: | - 65 Home Runs - 0.278 Batting Average`, - tgt: [ - { - contents: [ - { - items: [ - 'name', - { type: Type.MAP_VALUE, node: 'Mark McGwire' }, - 'accomplishment', - { - type: Type.MAP_VALUE, - node: 'Mark set a major league home run record in 1998.\n' - }, - 'stats', - { - type: Type.MAP_VALUE, - node: '65 Home Runs\n0.278 Batting Average\n' - } - ] - } - ] - } - ] - }, - - 'Example 2.17. Quoted Scalars': { - src: `unicode: "Sosa did fine.\\u263A" -control: "\\b1998\\t1999\\t2000\\n" -hex esc: "\\x0d\\x0a is \\r\\n" - -single: '"Howdy!" he cried.' -quoted: ' # Not a ''comment''.' -tie-fighter: '|\\-*-/|'`, - tgt: [ - { - contents: [ - { - items: [ - 'unicode', - { type: Type.MAP_VALUE, node: 'Sosa did fine.☺' }, - 'control', - { type: Type.MAP_VALUE, node: '\b1998\t1999\t2000\n' }, - 'hex esc', - { type: Type.MAP_VALUE, node: '\r\n is \r\n' }, - { type: Type.BLANK_LINE }, - 'single', - { type: Type.MAP_VALUE, node: '"Howdy!" he cried.' }, - 'quoted', - { type: Type.MAP_VALUE, node: " # Not a 'comment'." }, - 'tie-fighter', - { type: Type.MAP_VALUE, node: '|\\-*-/|' } - ] - } - ] - } - ] - }, - - 'Example 2.18. Multi-line Flow Scalars': { - src: `plain: - This unquoted scalar - spans many lines. - -quoted: "So does this - quoted scalar.\\n"`, - tgt: [ - { - contents: [ - { - items: [ - 'plain', - { - type: Type.MAP_VALUE, - node: 'This unquoted scalar spans many lines.' - }, - { type: Type.BLANK_LINE }, - 'quoted', - { type: Type.MAP_VALUE, node: 'So does this quoted scalar.\n' } - ] - } - ] - } - ] - } - }, - - '2.4. Tags': { - 'Example 2.19. Integers': { - src: `canonical: 12345 -decimal: +12345 -octal: 0o14 -hexadecimal: 0xC`, - tgt: [ - { - contents: [ - { - items: [ - 'canonical', - { type: Type.MAP_VALUE, node: '12345' }, - 'decimal', - { type: Type.MAP_VALUE, node: '+12345' }, - 'octal', - { type: Type.MAP_VALUE, node: '0o14' }, - 'hexadecimal', - { type: Type.MAP_VALUE, node: '0xC' } - ] - } - ] - } - ] - }, - - 'Example 2.20. Floating Point': { - src: `canonical: 1.23015e+3 -exponential: 12.3015e+02 -fixed: 1230.15 -negative infinity: -.inf -not a number: .NaN`, - tgt: [ - { - contents: [ - { - items: [ - 'canonical', - { type: Type.MAP_VALUE, node: '1.23015e+3' }, - 'exponential', - { type: Type.MAP_VALUE, node: '12.3015e+02' }, - 'fixed', - { type: Type.MAP_VALUE, node: '1230.15' }, - 'negative infinity', - { type: Type.MAP_VALUE, node: '-.inf' }, - 'not a number', - { type: Type.MAP_VALUE, node: '.NaN' } - ] - } - ] - } - ] - }, - - 'Example 2.21. Miscellaneous': { - src: `null: -booleans: [ true, false ] -string: '012345'`, - tgt: [ - { - contents: [ - { - items: [ - 'null', - { type: Type.MAP_VALUE, node: null }, - 'booleans', - { - type: Type.MAP_VALUE, - node: { items: ['[', 'true', ',', 'false', ']'] } - }, - 'string', - { type: Type.MAP_VALUE, node: '012345' } - ] - } - ] - } - ] - }, - - 'Example 2.22. Timestamps': { - src: `canonical: 2001-12-15T02:59:43.1Z -iso8601: 2001-12-14t21:59:43.10-05:00 -spaced: 2001-12-14 21:59:43.10 -5 -date: 2002-12-14`, - tgt: [ - { - contents: [ - { - items: [ - 'canonical', - { type: Type.MAP_VALUE, node: '2001-12-15T02:59:43.1Z' }, - 'iso8601', - { type: Type.MAP_VALUE, node: '2001-12-14t21:59:43.10-05:00' }, - 'spaced', - { type: Type.MAP_VALUE, node: '2001-12-14 21:59:43.10 -5' }, - 'date', - { type: Type.MAP_VALUE, node: '2002-12-14' } - ] - } - ] - } - ] - }, - - 'Example 2.23. Various Explicit Tags': { - src: `--- -not-date: !!str 2002-04-28 - -picture: !!binary | - R0lGODlhDAAMAIQAAP//9/X - 17unp5WZmZgAAAOfn515eXv - Pz7Y6OjuDg4J+fn5OTk6enp - 56enmleECcgggoBADs= - -application specific tag: !something | - The semantics of the tag - above may be different for - different documents.`, - tgt: [ - { - contents: [ - { - items: [ - 'not-date', - { - type: Type.MAP_VALUE, - node: { - tag: { handle: '!!', suffix: 'str' }, - strValue: '2002-04-28' - } - }, - { type: Type.BLANK_LINE }, - 'picture', - { - type: Type.MAP_VALUE, - node: { - tag: { handle: '!!', suffix: 'binary' }, - strValue: - 'R0lGODlhDAAMAIQAAP//9/X\n17unp5WZmZgAAAOfn515eXv\nPz7Y6OjuDg4J+fn5OTk6enp\n56enmleECcgggoBADs=\n' - } - }, - { type: Type.BLANK_LINE }, - 'application specific tag', - { - type: Type.MAP_VALUE, - node: { - tag: { handle: '!', suffix: 'something' }, - strValue: - 'The semantics of the tag\nabove may be different for\ndifferent documents.\n' - } - } - ] - } - ] - } - ] - }, - - 'Example 2.24. Global Tags': { - src: `%TAG ! tag:clarkevans.com,2002: ---- !shape - # Use the ! handle for presenting - # tag:clarkevans.com,2002:circle -- !circle - center: &ORIGIN {x: 73, y: 129} - radius: 7 -- !line - start: *ORIGIN - finish: { x: 89, y: 102 } -- !label - start: *ORIGIN - color: 0xFFEEBB - text: Pretty vector drawing.`, - tgt: [ - { - directives: [ - { name: 'TAG', parameters: ['!', 'tag:clarkevans.com,2002:'] } - ], - contents: [ - { - tag: { handle: '!', suffix: 'shape' }, - comment: - ' Use the ! handle for presenting\n tag:clarkevans.com,2002:circle', - items: [ - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!', suffix: 'circle' }, - items: [ - 'center', - { - type: Type.MAP_VALUE, - node: { - anchor: 'ORIGIN', - items: [ - '{', - 'x', - ':', - '73', - ',', - 'y', - ':', - '129', - '}' - ] - } - }, - 'radius', - { type: Type.MAP_VALUE, node: '7' } - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!', suffix: 'line' }, - items: [ - 'start', - { - type: Type.MAP_VALUE, - node: { type: Type.ALIAS, rawValue: 'ORIGIN' } - }, - 'finish', - { - type: Type.MAP_VALUE, - node: { - items: [ - '{', - 'x', - ':', - '89', - ',', - 'y', - ':', - '102', - '}' - ] - } - } - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!', suffix: 'label' }, - items: [ - 'start', - { - type: Type.MAP_VALUE, - node: { type: Type.ALIAS, rawValue: 'ORIGIN' } - }, - 'color', - { type: Type.MAP_VALUE, node: '0xFFEEBB' }, - 'text', - { type: Type.MAP_VALUE, node: 'Pretty vector drawing.' } - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 2.25. Unordered Sets': { - src: `# Sets are represented as a -# Mapping where each key is -# associated with a null value ---- !!set -? Mark McGwire -? Sammy Sosa -? Ken Griff`, - tgt: [ - { - directives: [ - { comment: ' Sets are represented as a' }, - { comment: ' Mapping where each key is' }, - { comment: ' associated with a null value' } - ], - contents: [ - { - tag: { handle: '!!', suffix: 'set' }, - items: [ - { type: Type.MAP_KEY, node: 'Mark McGwire' }, - { type: Type.MAP_KEY, node: 'Sammy Sosa' }, - { type: Type.MAP_KEY, node: 'Ken Griff' } - ] - } - ] - } - ] - }, - - 'Example 2.26. Ordered Mappings': { - src: `# Ordered maps are represented as -# A sequence of mappings, with -# each mapping having one key ---- !!omap -- Mark McGwire: 65 -- Sammy Sosa: 63 -- Ken Griffy: 58\n\n`, - tgt: [ - { - directives: [ - { comment: ' Ordered maps are represented as' }, - { comment: ' A sequence of mappings, with' }, - { comment: ' each mapping having one key' } - ], - contents: [ - { - tag: { handle: '!!', suffix: 'omap' }, - items: [ - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'Mark McGwire', - { type: Type.MAP_VALUE, node: '65' } - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: ['Sammy Sosa', { type: Type.MAP_VALUE, node: '63' }] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: ['Ken Griffy', { type: Type.MAP_VALUE, node: '58' }] - } - } - ] - } - ] - } - ] - } - }, - - '2.5. Full Length Example': { - 'Example 2.27. Invoice': { - src: `--- ! -invoice: 34843 -date : 2001-01-23 -bill-to: &id001 - given : Chris - family : Dumars - address: - lines: | - 458 Walkman Dr. - Suite #292 - city : Royal Oak - state : MI - postal : 48046 -ship-to: *id001 -product: - - sku : BL394D - quantity : 4 - description : Basketball - price : 450.00 - - sku : BL4438H - quantity : 1 - description : Super Hoop - price : 2392.00 -tax : 251.42 -total: 4443.52 -comments: - Late afternoon is best. - Backup contact is Nancy - Billsmer @ 338-4338.`, - tgt: [ - { - contents: [ - { - tag: { verbatim: 'tag:clarkevans.com,2002:invoice' }, - items: [ - 'invoice', - { type: Type.MAP_VALUE, node: '34843' }, - 'date', - { type: Type.MAP_VALUE, node: '2001-01-23' }, - 'bill-to', - { - type: Type.MAP_VALUE, - node: { - anchor: 'id001', - items: [ - 'given', - { type: Type.MAP_VALUE, node: 'Chris' }, - 'family', - { type: Type.MAP_VALUE, node: 'Dumars' }, - 'address', - { - type: Type.MAP_VALUE, - node: { - items: [ - 'lines', - { - type: Type.MAP_VALUE, - node: '458 Walkman Dr.\nSuite #292\n' - }, - 'city', - { type: Type.MAP_VALUE, node: 'Royal Oak' }, - 'state', - { type: Type.MAP_VALUE, node: 'MI' }, - 'postal', - { type: Type.MAP_VALUE, node: '48046' } - ] - } - } - ] - } - }, - 'ship-to', - { - type: Type.MAP_VALUE, - node: { type: Type.ALIAS, rawValue: 'id001' } - }, - 'product', - { - type: Type.MAP_VALUE, - node: { - items: [ - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'sku', - { type: Type.MAP_VALUE, node: 'BL394D' }, - 'quantity', - { type: Type.MAP_VALUE, node: '4' }, - 'description', - { type: Type.MAP_VALUE, node: 'Basketball' }, - 'price', - { type: Type.MAP_VALUE, node: '450.00' } - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'sku', - { type: Type.MAP_VALUE, node: 'BL4438H' }, - 'quantity', - { type: Type.MAP_VALUE, node: '1' }, - 'description', - { type: Type.MAP_VALUE, node: 'Super Hoop' }, - 'price', - { type: Type.MAP_VALUE, node: '2392.00' } - ] - } - } - ] - } - }, - 'tax', - { type: Type.MAP_VALUE, node: '251.42' }, - 'total', - { type: Type.MAP_VALUE, node: '4443.52' }, - 'comments', - { - type: Type.MAP_VALUE, - node: - 'Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338.' - } - ] - } - ] - } - ] - }, - - 'Example 2.28. Log File': { - src: `--- -Time: 2001-11-23 15:01:42 -5 -User: ed -Warning: - This is an error message - for the log file ---- -Time: 2001-11-23 15:02:31 -5 -User: ed -Warning: - A slightly different error - message. ---- -Date: 2001-11-23 15:03:17 -5 -User: ed -Fatal: - Unknown variable "bar" -Stack: - - file: TopClass.py - line: 23 - code: | - x = MoreObject("345\\n") - - file: MoreClass.py - line: 58 - code: |- - foo = bar\n`, - tgt: [ - { - contents: [ - { - items: [ - 'Time', - { type: Type.MAP_VALUE, node: '2001-11-23 15:01:42 -5' }, - 'User', - { type: Type.MAP_VALUE, node: 'ed' }, - 'Warning', - { - type: Type.MAP_VALUE, - node: 'This is an error message for the log file' - } - ] - } - ] - }, - { - contents: [ - { - items: [ - 'Time', - { type: Type.MAP_VALUE, node: '2001-11-23 15:02:31 -5' }, - 'User', - { type: Type.MAP_VALUE, node: 'ed' }, - 'Warning', - { - type: Type.MAP_VALUE, - node: 'A slightly different error message.' - } - ] - } - ] - }, - { - contents: [ - { - items: [ - 'Date', - { type: Type.MAP_VALUE, node: '2001-11-23 15:03:17 -5' }, - 'User', - { type: Type.MAP_VALUE, node: 'ed' }, - 'Fatal', - { type: Type.MAP_VALUE, node: 'Unknown variable "bar"' }, - 'Stack', - { - type: Type.MAP_VALUE, - node: { - items: [ - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'file', - { type: Type.MAP_VALUE, node: 'TopClass.py' }, - 'line', - { type: Type.MAP_VALUE, node: '23' }, - 'code', - { - type: Type.MAP_VALUE, - node: 'x = MoreObject("345\\n")\n' - } - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'file', - { type: Type.MAP_VALUE, node: 'MoreClass.py' }, - 'line', - { type: Type.MAP_VALUE, node: '58' }, - 'code', - { type: Type.MAP_VALUE, node: 'foo = bar' } - ] - } - } - ] - } - } - ] - } - ] - } - ] - } - }, - - '5.3. Indicator Characters': { - 'Example 5.3. Block Structure Indicators': { - src: `sequence: -- one -- two -mapping: - ? sky - : blue - sea : green`, - tgt: [ - { - contents: [ - { - items: [ - 'sequence', - { - type: Type.MAP_VALUE, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'one' }, - { type: Type.SEQ_ITEM, node: 'two' } - ] - } - }, - 'mapping', - { - type: Type.MAP_VALUE, - node: { - items: [ - { type: Type.MAP_KEY, node: 'sky' }, - { type: Type.MAP_VALUE, node: 'blue' }, - 'sea', - { type: Type.MAP_VALUE, node: 'green' } - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 5.4. Flow Collection Indicators': { - src: `sequence: [ one, two, ] -mapping: { sky: blue, sea: green }`, - tgt: [ - { - contents: [ - { - items: [ - 'sequence', - { - type: Type.MAP_VALUE, - node: { - items: ['[', 'one', ',', 'two', ',', ']'] - } - }, - 'mapping', - { - type: Type.MAP_VALUE, - node: { - items: [ - '{', - 'sky', - ':', - 'blue', - ',', - 'sea', - ':', - 'green', - '}' - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 5.5. Comment Indicator': { - src: `# Comment only.`, - tgt: [{ contents: [{ comment: ' Comment only.' }] }] - }, - - 'Example 5.6. Node Property Indicators': { - src: `anchored: !local &anchor value -alias: *anchor`, - tgt: [ - { - contents: [ - { - items: [ - 'anchored', - { - type: Type.MAP_VALUE, - node: { - tag: { handle: '!', suffix: 'local' }, - anchor: 'anchor', - strValue: 'value' - } - }, - 'alias', - { - type: Type.MAP_VALUE, - node: { type: Type.ALIAS, rawValue: 'anchor' } - } - ] - } - ] - } - ] - }, - - 'Example 5.7. Block Scalar Indicators': { - src: `literal: | - some - text -folded: > - some - text -`, - tgt: [ - { - contents: [ - { - items: [ - 'literal', - { type: Type.MAP_VALUE, node: 'some\ntext\n' }, - 'folded', - { type: Type.MAP_VALUE, node: 'some text\n' } - ] - } - ] - } - ] - }, - - 'Example 5.8. Quoted Scalar Indicators': { - src: `single: 'text' -double: "text"`, - tgt: [ - { - contents: [ - { - items: [ - 'single', - { type: Type.MAP_VALUE, node: 'text' }, - 'double', - { type: Type.MAP_VALUE, node: 'text' } - ] - } - ] - } - ] - }, - - 'Example 5.9. Directive Indicator': { - src: `%YAML 1.2 ---- text`, - tgt: [ - { - directives: [{ name: 'YAML', parameters: ['1.2'] }], - contents: ['text'] - } - ] - }, - - 'Example 5.10. Invalid use of Reserved Indicators': { - src: `commercial-at: @text -grave-accent: \`text`, - tgt: [ - { - contents: [ - { - items: [ - 'commercial-at', - { type: Type.MAP_VALUE, node: { strValue: { str: '@text' } } }, - 'grave-accent', - { type: Type.MAP_VALUE, node: { strValue: { str: '`text' } } } - ] - } - ] - } - ] - // ERROR: Reserved indicators can't start a plain scalar. - } - }, - '5.5. White Space Characters': { - 'Example 5.12. Tabs and Spaces': { - src: `# Tabs and spaces -quoted: "Quoted \t" -block:\t| - void main() { - \tprintf("Hello, world!\\n"); - }`, - tgt: [ - { - contents: [ - { comment: ' Tabs and spaces' }, - { - items: [ - 'quoted', - { type: Type.MAP_VALUE, node: 'Quoted \t' }, - 'block', - { - type: Type.MAP_VALUE, - node: 'void main() {\n\tprintf("Hello, world!\\n");\n}\n' - } - ] - } - ] - } - ] - } - }, - - '5.7. Escaped Characters': { - 'Example 5.13. Escaped Characters': { - src: `"Fun with \\\\ -\\" \\a \\b \\e \\f \\ -\\n \\r \\t \\v \\0 \\ -\\ \\_ \\N \\L \\P \\ -\\x41 \\u0041 \\U00000041"`, - tgt: [ - { - contents: [ - 'Fun with \x5C \x22 \x07 \x08 \x1B \x0C \x0A \x0D \x09 \x0B \x00 \x20 \xA0 \x85 \u2028 \u2029 A A A' - ] - } - ] - }, - - 'Example 5.14. Invalid Escaped Characters': { - src: `Bad escapes: - "\\c - \\xq-"`, - tgt: [ - { - contents: [ - { - items: [ - 'Bad escapes', - { type: Type.MAP_VALUE, node: { rawValue: '"\\c\n \\xq-"' } } - ] - } - ] - } - ] - // ERROR: c is an invalid escaped character. - // ERROR: q and - are invalid hex digits. - } - }, - - '6.1. Indentation Spaces': { - 'Example 6.1. Indentation Spaces': { - src: ` # Leading comment line spaces are - # neither content nor indentation. - -Not indented: - By one space: | - By four - spaces - Flow style: [ # Leading spaces - By two, # in flow style - Also by two, # are neither - \tStill by two # content nor - ] # indentation.`, - tgt: [ - { - contents: [ - { comment: ' Leading comment line spaces are' }, - { comment: ' neither content nor indentation.' }, - { type: Type.BLANK_LINE }, - { - items: [ - 'Not indented', - { - type: Type.MAP_VALUE, - node: { - items: [ - 'By one space', - { type: Type.MAP_VALUE, node: 'By four\n spaces\n' }, - 'Flow style', - { - type: Type.MAP_VALUE, - node: { - items: [ - '[', - { comment: ' Leading spaces' }, - 'By two', - ',', - { comment: ' in flow style' }, - 'Also by two', - ',', - { comment: ' are neither' }, - { - strValue: 'Still by two', - comment: ' content nor' - }, - ']' - ], - comment: ' indentation.' - } - } - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 6.2. Indentation Indicators': { - src: `? a -: -\tb - - -\tc - - d`, - tgt: [ - { - contents: [ - { - items: [ - { type: Type.MAP_KEY, node: 'a' }, - { - type: Type.MAP_VALUE, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'b' }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'c' }, - { type: Type.SEQ_ITEM, node: 'd' } - ] - } - } - ] - } - } - ] - } - ] - } - ] - } - }, - - '6.2. Separation Spaces': { - 'Example 6.3. Separation Spaces': { - src: `- foo:\t bar -- - baz - -\tbaz`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { - items: ['foo', { type: Type.MAP_VALUE, node: 'bar' }] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'baz' }, - { type: Type.SEQ_ITEM, node: 'baz' } - ] - } - } - ] - } - ] - } - ] - } - }, - - '6.3. Line Prefixes': { - 'Example 6.4. Line Prefixes': { - src: `plain: text - lines -quoted: "text - \tlines" -block: | - text - \tlines`, - tgt: [ - { - contents: [ - { - items: [ - 'plain', - { type: Type.MAP_VALUE, node: 'text lines' }, - 'quoted', - { type: Type.MAP_VALUE, node: 'text lines' }, - 'block', - { type: Type.MAP_VALUE, node: 'text\n \tlines\n' } - ] - } - ] - } - ] - } - }, - - '6.4. Empty Lines': { - 'Example 6.5. Empty Lines': { - src: `Folding: - "Empty line - \t - as a line feed" -Chomping: | - Clipped empty lines - `, - tgt: [ - { - contents: [ - { - items: [ - 'Folding', - { type: Type.MAP_VALUE, node: 'Empty line\nas a line feed' }, - 'Chomping', - { type: Type.MAP_VALUE, node: 'Clipped empty lines\n' } - ] - } - ] - } - ] - } - }, - - '6.5. Line Folding': { - 'Example 6.6. Line Folding': { - src: `>- - trimmed -·· -· - -··as -··space`.replace(/·/g, ' '), - tgt: [{ contents: ['trimmed\n\n\nas space'] }] - }, - - 'Example 6.7. Block Folding': { - src: `> -··foo· -· -··\t·bar - -··baz\n`.replace(/·/g, ' '), - tgt: [{ contents: ['foo \n\n\t bar\n\nbaz\n'] }] - }, - - 'Example 6.8. Flow Folding': { - src: `" - foo\t -\t - \t bar - - baz -"`, - tgt: [{ contents: [' foo\nbar\nbaz '] }] - } - }, - - '6.6. Comments': { - 'Example 6.9. Separated Comment': { - src: `key: # Comment - value`, - tgt: [ - { - contents: [ - { - items: [ - 'key', - { type: Type.MAP_VALUE, comment: ' Comment', node: 'value' } - ] - } - ] - } - ] - }, - - 'Example 6.10. Comment Lines': { - src: ` # Comment - \n\n`, - tgt: [{ contents: [{ comment: ' Comment' }] }] - }, - - 'Example 6.11. Multi-Line Comments': { - src: `key: # Comment - # lines - value\n`, - tgt: [ - { - contents: [ - { - items: [ - 'key', - { - type: Type.MAP_VALUE, - comment: ' Comment\n lines', - node: 'value' - } - ] - } - ] - } - ] - } - }, - '6.7. Separation Lines': { - 'Example 6.12. Separation Spaces': { - src: `{ first: Sammy, last: Sosa }: -# Statistics: - hr: # Home runs - 65 - avg: # Average - 0.278`, - tgt: [ - { - contents: [ - { - items: [ - { - items: [ - '{', - 'first', - ':', - 'Sammy', - ',', - 'last', - ':', - 'Sosa', - '}' - ] - }, - { - type: Type.MAP_VALUE, - comment: ' Statistics:', - node: { - items: [ - 'hr', - { - type: Type.MAP_VALUE, - comment: ' Home runs', - node: '65' - }, - 'avg', - { - type: Type.MAP_VALUE, - comment: ' Average', - node: '0.278' - } - ] - } - } - ] - } - ] - } - ] - } - }, - '6.8. Directives': { - 'Example 6.13. Reserved Directives': { - src: `%FOO bar baz # Should be ignored -# with a warning. ---- "foo"`, - tgt: [ - { - directives: [ - { - name: 'FOO', - parameters: ['bar', 'baz'], - comment: ' Should be ignored' - }, - { comment: ' with a warning.' } - ], - contents: ['foo'] - } - ] - } - }, - '6.8.1. “YAML” Directives': { - 'Example 6.14. “YAML” directive': { - src: `%YAML 1.3 # Attempt parsing - # with a warning ---- -"foo"`, - tgt: [ - { - directives: [ - { name: 'YAML', parameters: ['1.3'], comment: ' Attempt parsing' }, - { comment: ' with a warning' } - ], - contents: ['foo'] - } - ] - }, - - 'Example 6.15. Invalid Repeated YAML directive': { - src: `%YAML 1.2 -%YAML 1.1 -foo`, - tgt: [ - { - directives: [ - { name: 'YAML', parameters: ['1.2'] }, - { name: 'YAML', parameters: ['1.1'] } - ], - contents: ['foo'] - } - ] - // ERROR: The YAML directive must only be given at most once per document. - } - }, - '6.8.2. “TAG” Directives': { - 'Example 6.16. “TAG” directive': { - src: `%TAG !yaml! tag:yaml.org,2002: ---- -!yaml!str "foo"`, - tgt: [ - { - directives: [ - { name: 'TAG', parameters: ['!yaml!', 'tag:yaml.org,2002:'] } - ], - contents: [ - { tag: { handle: '!yaml!', suffix: 'str' }, strValue: 'foo' } - ] - } - ] - }, - - 'Example 6.17. Invalid Repeated TAG directive': { - src: `%TAG ! !foo -%TAG ! !foo -bar`, - tgt: [ - { - directives: [ - { name: 'TAG', parameters: ['!', '!foo'] }, - { name: 'TAG', parameters: ['!', '!foo'] } - ], - contents: ['bar'] - } - ] - // ERROR: The TAG directive must only be given at most once per handle in the same document. - }, - - 'Example 6.18. Primary Tag Handle': { - src: `# Private -!foo "bar" -... -# Global -%TAG ! tag:example.com,2000:app/ ---- -!foo "bar"`, - tgt: [ - { - contents: [ - { comment: ' Private' }, - { tag: { handle: '!', suffix: 'foo' }, strValue: 'bar' } - ] - }, - { - directives: [ - { comment: ' Global' }, - { name: 'TAG', parameters: ['!', 'tag:example.com,2000:app/'] } - ], - contents: [{ tag: { handle: '!', suffix: 'foo' }, strValue: 'bar' }] - } - ] - }, - - 'Example 6.19. Secondary Tag Handle': { - src: `%TAG !! tag:example.com,2000:app/ ---- -!!int 1 - 3 # Interval, not integer`, - tgt: [ - { - directives: [ - { name: 'TAG', parameters: ['!!', 'tag:example.com,2000:app/'] } - ], - contents: [ - { - tag: { handle: '!!', suffix: 'int' }, - strValue: '1 - 3', - comment: ' Interval, not integer' - } - ] - } - ] - }, - - 'Example 6.20. Tag Handles': { - src: `%TAG !e! tag:example.com,2000:app/ ---- -!e!foo "bar"`, - tgt: [ - { - directives: [ - { name: 'TAG', parameters: ['!e!', 'tag:example.com,2000:app/'] } - ], - contents: [{ tag: { handle: '!e!', suffix: 'foo' }, strValue: 'bar' }] - } - ] - }, - - 'Example 6.21. Local Tag Prefix': { - src: `%TAG !m! !my- ---- # Bulb here -!m!light fluorescent -... -%TAG !m! !my- ---- # Color here -!m!light green`, - tgt: [ - { - directives: [{ name: 'TAG', parameters: ['!m!', '!my-'] }], - contents: [ - { comment: ' Bulb here' }, - { - tag: { handle: '!m!', suffix: 'light' }, - strValue: 'fluorescent' - } - ] - }, - { - directives: [{ name: 'TAG', parameters: ['!m!', '!my-'] }], - contents: [ - { comment: ' Color here' }, - { tag: { handle: '!m!', suffix: 'light' }, strValue: 'green' } - ] - } - ] - }, - - 'Example 6.22. Global Tag Prefix': { - src: `%TAG !e! tag:example.com,2000:app/ ---- -- !e!foo "bar"`, - tgt: [ - { - directives: [ - { name: 'TAG', parameters: ['!e!', 'tag:example.com,2000:app/'] } - ], - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!e!', suffix: 'foo' }, - strValue: 'bar' - } - } - ] - } - ] - } - ] - } - }, - '6.9. Node Properties': { - 'Example 6.23. Node Properties': { - src: `!!str &a1 "foo": - !!str bar -&a2 baz : *a1`, - tgt: [ - { - contents: [ - { - items: [ - { - tag: { handle: '!!', suffix: 'str' }, - anchor: 'a1', - strValue: 'foo' - }, - { - type: Type.MAP_VALUE, - node: { - tag: { handle: '!!', suffix: 'str' }, - strValue: 'bar' - } - }, - { anchor: 'a2', strValue: 'baz' }, - { - type: Type.MAP_VALUE, - node: { type: Type.ALIAS, rawValue: 'a1' } - } - ] - } - ] - } - ] - }, - - 'Example 6.24. Verbatim Tags': { - src: `! foo : - ! baz`, - tgt: [ - { - contents: [ - { - items: [ - { tag: { verbatim: 'tag:yaml.org,2002:str' }, strValue: 'foo' }, - { - type: Type.MAP_VALUE, - node: { tag: { verbatim: '!bar' }, strValue: 'baz' } - } - ] - } - ] - } - ] - }, - - 'Example 6.25. Invalid Verbatim Tags': { - src: `- ! foo -- !<$:?> bar`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { tag: { verbatim: '!' }, strValue: 'foo' } - }, - { - type: Type.SEQ_ITEM, - node: { tag: { verbatim: '$:?' }, strValue: 'bar' } - } - ] - } - ] - } - ] - // ERROR: Verbatim tags aren't resolved, so ! is invalid. - // ERROR: The $:? tag is neither a global URI tag nor a local tag starting with “!”. - }, - - 'Example 6.26. Tag Shorthands': { - src: `%TAG !e! tag:example.com,2000:app/ ---- -- !local foo -- !!str bar -- !e!tag%21 baz`, - tgt: [ - { - directives: [ - { name: 'TAG', parameters: ['!e!', 'tag:example.com,2000:app/'] } - ], - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!', suffix: 'local' }, - strValue: 'foo' - } - }, - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!!', suffix: 'str' }, - strValue: 'bar' - } - }, - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!e!', suffix: 'tag%21' }, - strValue: 'baz' - } - } - ] - } - ] - } - ] - }, - - 'Example 6.27. Invalid Tag Shorthands': { - src: `%TAG !e! tag:example,2000:app/ ---- -- !e! foo -- !h!bar baz`, - tgt: [ - { - directives: [ - { name: 'TAG', parameters: ['!e!', 'tag:example,2000:app/'] } - ], - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { tag: { handle: '!e!', suffix: '' }, strValue: 'foo' } - }, - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!h!', suffix: 'bar' }, - strValue: 'baz' - } - } - ] - } - ] - } - ] - // ERROR: The !e! handle has no suffix. - // ERROR: The !h! handle wasn't declared. - }, - - 'Example 6.28. Non-Specific Tags': { - src: `# Assuming conventional resolution: -- "12" -- 12 -- ! 12`, - tgt: [ - { - contents: [ - { comment: ' Assuming conventional resolution:' }, - { - items: [ - { type: Type.SEQ_ITEM, node: '12' }, - { type: Type.SEQ_ITEM, node: '12' }, - { - type: Type.SEQ_ITEM, - node: { tag: { handle: '!', suffix: '' }, strValue: '12' } - } - ] - } - ] - } - ] - }, - - 'Example 6.29. Node Anchors': { - src: `First occurrence: &anchor Value -Second occurrence: *anchor`, - tgt: [ - { - contents: [ - { - items: [ - 'First occurrence', - { - type: Type.MAP_VALUE, - node: { anchor: 'anchor', strValue: 'Value' } - }, - 'Second occurrence', - { - type: Type.MAP_VALUE, - node: { type: Type.ALIAS, rawValue: 'anchor' } - } - ] - } - ] - } - ] - } - }, - - '7.1. Alias Nodes': { - 'Example 7.1. Alias Nodes': { - src: `First occurrence: &anchor Foo -Second occurrence: *anchor -Override anchor: &anchor Bar -Reuse anchor: *anchor`, - tgt: [ - { - contents: [ - { - items: [ - 'First occurrence', - { - type: Type.MAP_VALUE, - node: { anchor: 'anchor', strValue: 'Foo' } - }, - 'Second occurrence', - { - type: Type.MAP_VALUE, - node: { type: Type.ALIAS, rawValue: 'anchor' } - }, - 'Override anchor', - { - type: Type.MAP_VALUE, - node: { anchor: 'anchor', strValue: 'Bar' } - }, - 'Reuse anchor', - { - type: Type.MAP_VALUE, - node: { type: Type.ALIAS, rawValue: 'anchor' } - } - ] - } - ] - } - ] - } - }, - - '7.2. Empty Nodes': { - 'Example 7.2. Empty Content': { - src: `{ - foo : !!str, - !!str : bar, -}`, - tgt: [ - { - contents: [ - { - items: [ - '{', - 'foo', - ':', - { tag: { handle: '!!', suffix: 'str' } }, - ',', - { tag: { handle: '!!', suffix: 'str' } }, - ':', - 'bar', - ',', - '}' - ] - } - ] - } - ] - }, - - 'Example 7.3. Completely Empty Flow Nodes': { - src: `{ - ? foo :, - : bar, -}`, - tgt: [ - { - contents: [ - { items: ['{', '?', 'foo', ':', ',', ':', 'bar', ',', '}'] } - ] - } - ] - } - }, - - '7.3.1. Double-Quoted Style': { - 'Example 7.4. Double Quoted Implicit Keys': { - src: `"implicit block key" : [ - "implicit flow key" : value, - ]`, - tgt: [ - { - contents: [ - { - items: [ - 'implicit block key', - { - type: Type.MAP_VALUE, - node: { - items: ['[', 'implicit flow key', ':', 'value', ',', ']'] - } - } - ] - } - ] - } - ] - }, - - 'Example 7.5. Double Quoted Line Breaks': { - src: `"folded -to a space,\t - -to a line feed, or \t\\ - \\ \tnon-content"`, - tgt: [ - { - contents: ['folded to a space,\nto a line feed, or \t \tnon-content'] - } - ] - }, - - 'Example 7.6. Double Quoted Lines': { - src: `" 1st non-empty - - 2nd non-empty -\t3rd non-empty "`, - tgt: [{ contents: [' 1st non-empty\n2nd non-empty 3rd non-empty '] }] - } - }, - - '7.3.2. Single-Quoted Style': { - 'Example 7.7. Single Quoted Characters': { - src: ` 'here''s to "quotes"'`, - tgt: [{ contents: ['here\'s to "quotes"'] }] - }, - - 'Example 7.8. Single Quoted Implicit Keys': { - src: `'implicit block key' : [ - 'implicit flow key' : value, - ]`, - tgt: [ - { - contents: [ - { - items: [ - 'implicit block key', - { - type: Type.MAP_VALUE, - node: { - items: ['[', 'implicit flow key', ':', 'value', ',', ']'] - } - } - ] - } - ] - } - ] - }, - - 'Example 7.9. Single Quoted Lines': { - src: `' 1st non-empty - - 2nd non-empty\t -\t3rd non-empty '`, - tgt: [{ contents: [' 1st non-empty\n2nd non-empty 3rd non-empty '] }] - } - }, - - '7.3.3. Plain Style': { - 'Example 7.10. Plain Characters': { - src: `# Outside flow collection: -- ::vector -- ": - ()" -- Up, up, and away! -- -123 -- http://example.com/foo#bar -# Inside flow collection: -- [ ::vector, - ": - ()", - "Up, up and away!", - -123, - http://example.com/foo#bar ]`, - tgt: [ - { - contents: [ - { comment: ' Outside flow collection:' }, - { - items: [ - { type: Type.SEQ_ITEM, node: '::vector' }, - { type: Type.SEQ_ITEM, node: ': - ()' }, - { type: Type.SEQ_ITEM, node: 'Up, up, and away!' }, - { type: Type.SEQ_ITEM, node: '-123' }, - { type: Type.SEQ_ITEM, node: 'http://example.com/foo#bar' }, - { comment: ' Inside flow collection:' }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - '[', - '::vector', - ',', - ': - ()', - ',', - 'Up, up and away!', - ',', - '-123', - ',', - 'http://example.com/foo#bar', - ']' - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 7.11. Plain Implicit Keys': { - src: `implicit block key : [ - implicit flow key : value, - ]`, - tgt: [ - { - contents: [ - { - items: [ - 'implicit block key', - { - type: Type.MAP_VALUE, - node: { - items: ['[', 'implicit flow key', ':', 'value', ',', ']'] - } - } - ] - } - ] - } - ] - }, - - 'Example 7.12. Plain Lines': { - src: `1st non-empty - - 2nd non-empty -\t3rd non-empty`, - tgt: [{ contents: ['1st non-empty\n2nd non-empty 3rd non-empty'] }] - } - }, - - '7.4.1. Flow Sequences': { - 'Example 7.13. Flow Sequence': { - src: `- [ one, two, ] -- [three ,four]`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { items: ['[', 'one', ',', 'two', ',', ']'] } - }, - { - type: Type.SEQ_ITEM, - node: { items: ['[', 'three', ',', 'four', ']'] } - } - ] - } - ] - } - ] - }, - - 'Example 7.14. Flow Sequence Entries': { - src: `[ -"double - quoted", 'single - quoted', -plain - text, [ nested ], -single: pair, -]`, - tgt: [ - { - contents: [ - { - items: [ - '[', - 'double quoted', - ',', - 'single quoted', - ',', - 'plain text', - ',', - { items: ['[', 'nested', ']'] }, - ',', - 'single', - ':', - 'pair', - ',', - ']' - ] - } - ] - } - ] - } - }, - - '7.4.2. Flow Mappings': { - 'Example 7.15. Flow Mappings': { - src: `- { one : two , three: four , } -- {five: six,seven : eight}`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { - items: [ - '{', - 'one', - ':', - 'two', - ',', - 'three', - ':', - 'four', - ',', - '}' - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - '{', - 'five', - ':', - 'six', - ',', - 'seven', - ':', - 'eight', - '}' - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 7.16. Flow Mapping Entries': { - src: `{ -? explicit: entry, -implicit: entry, -? -}`, - tgt: [ - { - contents: [ - { - items: [ - '{', - '?', - 'explicit', - ':', - 'entry', - ',', - 'implicit', - ':', - 'entry', - ',', - '?', - '}' - ] - } - ] - } - ] - }, - - 'Example 7.17. Flow Mapping Separate Values': { - src: `{ -unquoted : "separate", -http://foo.com, -omitted value:, -: omitted key, -}`, - tgt: [ - { - contents: [ - { - items: [ - '{', - 'unquoted', - ':', - 'separate', - ',', - 'http://foo.com', - ',', - 'omitted value', - ':', - ',', - ':', - 'omitted key', - ',', - '}' - ] - } - ] - } - ] - }, - - 'Example 7.18. Flow Mapping Adjacent Values': { - src: `{ -"adjacent":value, -"readable": value, -"empty": -}`, - tgt: [ - { - contents: [ - { - items: [ - '{', - 'adjacent', - ':', - 'value', - ',', - 'readable', - ':', - 'value', - ',', - 'empty', - ':', - '}' - ] - } - ] - } - ] - }, - - 'Example 7.19. Single Pair Flow Mappings': { - src: `[ -foo: bar -]`, - tgt: [{ contents: [{ items: ['[', 'foo', ':', 'bar', ']'] }] }] - }, - - 'Example 7.20. Single Pair Explicit Entry': { - src: `[ -? foo - bar : baz -]`, - tgt: [{ contents: [{ items: ['[', '?', 'foo bar', ':', 'baz', ']'] }] }] - }, - - 'Example 7.21. Single Pair Implicit Entries': { - src: `- [ YAML : separate ] -- [ : empty key entry ] -- [ {JSON: like}:adjacent ]`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { items: ['[', 'YAML', ':', 'separate', ']'] } - }, - { - type: Type.SEQ_ITEM, - node: { items: ['[', ':', 'empty key entry', ']'] } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - '[', - { items: ['{', 'JSON', ':', 'like', '}'] }, - ':', - 'adjacent', - ']' - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 7.22. Invalid Implicit Keys': { - src: `[ foo - bar: invalid, - "foo...>1K characters...bar": invalid ]`, - tgt: [ - { - contents: [ - { - items: [ - '[', - 'foo bar', - ':', - 'invalid', - ',', - 'foo...>1K characters...bar', - ':', - 'invalid', - ']' - ] - } - ] - } - ] - // ERROR: The foo bar key spans multiple lines - // ERROR: The foo...bar key is too long - } - }, - - '7.5. Flow Nodes': { - 'Example 7.23. Flow Content': { - src: `- [ a, b ] -- { a: b } -- "a" -- 'b' -- c`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { items: ['[', 'a', ',', 'b', ']'] } - }, - { - type: Type.SEQ_ITEM, - node: { items: ['{', 'a', ':', 'b', '}'] } - }, - { type: Type.SEQ_ITEM, node: 'a' }, - { type: Type.SEQ_ITEM, node: 'b' }, - { type: Type.SEQ_ITEM, node: 'c' } - ] - } - ] - } - ] - }, - - 'Example 7.24. Flow Nodes': { - src: `- !!str "a" -- 'b' -- &anchor "c" -- *anchor -- !!str`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { tag: { handle: '!!', suffix: 'str' }, strValue: 'a' } - }, - { type: Type.SEQ_ITEM, node: 'b' }, - { - type: Type.SEQ_ITEM, - node: { anchor: 'anchor', strValue: 'c' } - }, - { - type: Type.SEQ_ITEM, - node: { type: Type.ALIAS, rawValue: 'anchor' } - }, - { - type: Type.SEQ_ITEM, - node: { tag: { handle: '!!', suffix: 'str' } } - } - ] - } - ] - } - ] - } - }, - - '8.1.1. Block Scalar Headers': { - 'Example 8.1. Block Scalar Header': { - src: `- | # Empty header - literal -- >1 # Indentation indicator - folded -- |+ # Chomping indicator - keep - -- >1- # Both indicators - strip`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { comment: ' Empty header', strValue: 'literal\n' } - }, - { - type: Type.SEQ_ITEM, - node: { - comment: ' Indentation indicator', - strValue: ' folded\n' - } - }, - { - type: Type.SEQ_ITEM, - node: { - comment: ' Chomping indicator', - strValue: 'keep\n\n' - } - }, - { - type: Type.SEQ_ITEM, - node: { comment: ' Both indicators', strValue: ' strip' } - } - ] - } - ] - } - ] - }, - - 'Example 8.2. Block Indentation Indicator': { - src: `- | -·detected -- > -· -·· -··# detected -- |1 -··explicit -- > -·\t -·detected`.replace(/·/g, ' '), - tgt: [ - { - contents: [ - { - items: [ - { type: Type.SEQ_ITEM, node: 'detected\n' }, - { type: Type.SEQ_ITEM, node: '\n\n# detected\n' }, - { type: Type.SEQ_ITEM, node: ' explicit\n' }, - { type: Type.SEQ_ITEM, node: '\t\ndetected\n' } - ] - } - ] - } - ] - }, - - 'Example 8.3. Invalid Block Scalar Indentation Indicators': { - src: `- | -·· -·text ---- -- > -··text -·text ---- -- |2 -·text`.replace(/·/g, ' '), - tgt: [ - { - contents: [{ items: [{ node: ' \ntext\n' }] }] - }, - { - contents: [{ items: [{ node: 'text text\n' }] }] - }, - { - contents: [{ items: [{ node: 'text\n' }] }] - } - ] - // ERROR: A leading all-space line must not have too many spaces. - // ERROR: A following text line must not be less indented. - // ERROR: The text is less indented than the indicated level. - }, - - 'Example 8.4. Chomping Final Line Break': { - src: `strip: |- - text -clip: | - text -keep: |+ - text\n`, - tgt: [ - { - contents: [ - { - items: [ - 'strip', - { type: Type.MAP_VALUE, node: 'text' }, - 'clip', - { type: Type.MAP_VALUE, node: 'text\n' }, - 'keep', - { type: Type.MAP_VALUE, node: 'text\n' } - ] - } - ] - } - ] - }, - - 'Example 8.5. Chomping Trailing Lines': { - src: ` - # Strip - # Comments: -strip: |- - # text - - # Clip - # comments: - -clip: | - # text - - # Keep - # comments: - -keep: |+ - # text - - # Trail - # comments.`, - tgt: [ - { - contents: [ - { type: Type.BLANK_LINE }, - { comment: ' Strip' }, - { comment: ' Comments:' }, - { - items: [ - 'strip', - { type: Type.MAP_VALUE, node: '# text' }, - { type: Type.BLANK_LINE }, - { comment: ' Clip' }, - { comment: ' comments:' }, - { type: Type.BLANK_LINE }, - 'clip', - { type: Type.MAP_VALUE, node: '# text\n' }, - { type: Type.BLANK_LINE }, - { comment: ' Keep' }, - { comment: ' comments:' }, - { type: Type.BLANK_LINE }, - 'keep', - { type: Type.MAP_VALUE, node: '# text\n\n' } - ] - }, - { comment: ' Trail' }, - { comment: ' comments.' } - ] - } - ] - }, - - 'Example 8.6. Empty Scalar Chomping': { - src: `strip: >- - -clip: > - -keep: |+\n\n`, - tgt: [ - { - contents: [ - { - items: [ - 'strip', - { type: Type.MAP_VALUE, node: '' }, - { type: Type.BLANK_LINE }, - 'clip', - { type: Type.MAP_VALUE, node: '' }, - { type: Type.BLANK_LINE }, - 'keep', - { type: Type.MAP_VALUE, node: '\n' } - ] - } - ] - } - ] - } - }, - - '8.1.2. Literal Style': { - 'Example 8.7. Literal Scalar': { - src: `| - literal - \ttext\n\n`, - tgt: [{ contents: ['literal\n\ttext\n'] }] - }, - - 'Example 8.8. Literal Content': { - src: `| -· -·· -··literal -··· -·· -··text - -·# Comment`.replace(/·/g, ' '), - tgt: [ - { - contents: ['\n\nliteral\n \n\ntext\n', { comment: ' Comment' }] - } - ] - } - }, - - '8.1.3. Folded Style': { - 'Example 8.9. Folded Scalar': { - src: `> - folded - text\n\n`, - tgt: [{ contents: ['folded text\n'] }] - }, - - 'Example 8.10. Folded Lines': { - src: `> - - folded - line - - next - line - * bullet - - * list - * lines - - last - line - -# Comment`, - tgt: [ - { - contents: [ - '\nfolded line\nnext line\n * bullet\n\n * list\n * lines\n\nlast line\n', - { comment: ' Comment' } - ] - } - ] - } - }, - - '8.2.1. Block Sequences': { - 'Example 8.14. Block Sequence': { - src: `block sequence: - - one - - two : three\n`, - tgt: [ - { - contents: [ - { - items: [ - 'block sequence', - { - type: Type.MAP_VALUE, - node: { - items: [ - { type: Type.SEQ_ITEM, node: 'one' }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'two', - { type: Type.MAP_VALUE, node: 'three' } - ] - } - } - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 8.15. Block Sequence Entry Types': { - src: `- # Empty -- | - block node -- - one # Compact - - two # sequence -- one: two # Compact mapping`, - tgt: [ - { - contents: [ - { - items: [ - { type: Type.SEQ_ITEM, node: null, comment: ' Empty' }, - { type: Type.SEQ_ITEM, node: 'block node\n' }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - { - type: Type.SEQ_ITEM, - node: { comment: ' Compact', strValue: 'one' } - }, - { - type: Type.SEQ_ITEM, - node: { comment: ' sequence', strValue: 'two' } - } - ] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - 'one', - { - type: Type.MAP_VALUE, - node: { comment: ' Compact mapping', strValue: 'two' } - } - ] - } - } - ] - } - ] - } - ] - } - }, - - '8.2.2. Block Mappings': { - 'Example 8.16. Block Mappings': { - src: `block mapping: - key: value\n`, - tgt: [ - { - contents: [ - { - items: [ - 'block mapping', - { - type: Type.MAP_VALUE, - node: { - items: ['key', { type: Type.MAP_VALUE, node: 'value' }] - } - } - ] - } - ] - } - ] - }, - - 'Example 8.17. Explicit Block Mapping Entries': { - src: `? explicit key # Empty value -? | - block key -: - one # Explicit compact - - two # block value\n`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.MAP_KEY, - node: { comment: ' Empty value', strValue: 'explicit key' } - }, - { type: Type.MAP_KEY, node: 'block key\n' }, - { - type: Type.MAP_VALUE, - node: { - items: [ - { - type: Type.SEQ_ITEM, - node: { comment: ' Explicit compact', strValue: 'one' } - }, - { - type: Type.SEQ_ITEM, - node: { comment: ' block value', strValue: 'two' } - } - ] - } - } - ] - } - ] - } - ] - }, - - 'Example 8.18. Implicit Block Mapping Entries': { - src: `plain key: in-line value -: # Both empty -"quoted key": -- entry`, - tgt: [ - { - contents: [ - { - items: [ - 'plain key', - { type: Type.MAP_VALUE, node: 'in-line value' }, - { type: Type.MAP_VALUE, node: null, comment: ' Both empty' }, - 'quoted key', - { - type: Type.MAP_VALUE, - node: { - items: [{ type: Type.SEQ_ITEM, node: 'entry' }] - } - } - ] - } - ] - } - ] - }, - - 'Example 8.19. Compact Block Mappings': { - src: `- sun: yellow -- ? earth: blue - : moon: white\n`, - tgt: [ - { - contents: [ - { - items: [ - { - type: Type.SEQ_ITEM, - node: { - items: ['sun', { type: Type.MAP_VALUE, node: 'yellow' }] - } - }, - { - type: Type.SEQ_ITEM, - node: { - items: [ - { - type: Type.MAP_KEY, - node: { - items: [ - 'earth', - { type: Type.MAP_VALUE, node: 'blue' } - ] - } - }, - { - type: Type.MAP_VALUE, - node: { - items: [ - 'moon', - { type: Type.MAP_VALUE, node: 'white' } - ] - } - } - ] - } - } - ] - } - ] - } - ] - } - }, - - '8.2.3. Block Nodes': { - 'Example 8.20. Block Types': { - src: `- - "flow in block" -- > - Block scalar -- !!map # Block collection - foo : bar\n`, - tgt: [ - { - contents: [ - { - items: [ - { type: Type.SEQ_ITEM, node: 'flow in block' }, - { type: Type.SEQ_ITEM, node: 'Block scalar\n' }, - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!!', suffix: 'map' }, - comment: ' Block collection', - items: ['foo', { type: Type.MAP_VALUE, node: 'bar' }] - } - } - ] - } - ] - } - ] - }, - - 'Example 8.21. Block Scalar Nodes': { - src: `literal: |2 - value -folded: - !foo - >1 - value`, - tgt: [ - { - contents: [ - { - items: [ - 'literal', - { type: Type.MAP_VALUE, node: 'value\n' }, - 'folded', - { - type: Type.MAP_VALUE, - node: { - tag: { handle: '!', suffix: 'foo' }, - strValue: 'value\n' - } - } // trailing \n against spec - ] - } - ] - } - ] - }, - - 'Example 8.22. Block Collection Nodes': { - src: `sequence: !!seq -- entry -- !!seq - - nested -mapping: !!map - foo: bar`, - tgt: [ - { - contents: [ - { - items: [ - 'sequence', - { - type: Type.MAP_VALUE, - node: { - tag: { handle: '!!', suffix: 'seq' }, - items: [ - { type: Type.SEQ_ITEM, node: 'entry' }, - { - type: Type.SEQ_ITEM, - node: { - tag: { handle: '!!', suffix: 'seq' }, - items: [{ type: Type.SEQ_ITEM, node: 'nested' }] - } - } - ] - } - }, - 'mapping', - { - type: Type.MAP_VALUE, - node: { - tag: { handle: '!!', suffix: 'map' }, - items: ['foo', { type: Type.MAP_VALUE, node: 'bar' }] - } - } - ] - } - ] - } - ] - } - }, - - '9.1. Documents': { - 'Example 9.1. Document Prefix': { - src: `\u{FEFF}# Comment -# lines -Document`, - tgt: [ - { - contents: [{ comment: ' Comment' }, { comment: ' lines' }, 'Document'] - } - ] - }, - - 'Example 9.2. Document Markers': { - src: `%YAML 1.2 ---- -Document -... # Suffix`, - tgt: [ - { - directives: [{ name: 'YAML', parameters: ['1.2'] }], - contents: ['Document', { comment: ' Suffix' }] - } - ] - }, - - 'Example 9.3. Bare Documents': { - src: `Bare -document -... -# No document -... -| -%!PS-Adobe-2.0 # Not the first line`, - tgt: [ - { - contents: ['Bare document'] - }, - { - contents: [{ comment: ' No document' }] - }, - { - contents: ['%!PS-Adobe-2.0 # Not the first line\n'] - } - ] - }, - - 'Example 9.4. Explicit Documents': { - src: `--- -{ matches -% : 20 } -... ---- -# Empty -...`, - tgt: [ - { - contents: [{ items: ['{', 'matches %', ':', '20', '}'] }] - }, - { - contents: [{ comment: ' Empty' }] - } - ] - }, - - 'Example 9.5. Directives Documents': { - src: `%YAML 1.2 ---- | -%!PS-Adobe-2.0 -... -%YAML 1.2 ---- -# Empty -...`, - tgt: [ - { - directives: [{ name: 'YAML', parameters: ['1.2'] }], - contents: ['%!PS-Adobe-2.0\n'] - }, - { - directives: [{ name: 'YAML', parameters: ['1.2'] }], - contents: [{ comment: ' Empty' }] - } - ] - } - }, - - '9.2. Streams': { - 'Example 9.6. Stream': { - src: `Document ---- -# Empty -... -%YAML 1.2 ---- -matches %: 20`, - tgt: [ - { - contents: ['Document'] - }, - { - contents: [{ comment: ' Empty' }] - }, - { - directives: [{ name: 'YAML', parameters: ['1.2'] }], - contents: [ - { items: ['matches %', { type: Type.MAP_VALUE, node: '20' }] } - ] - } - ] - } - }, - - 'yaml-test-suite': { - '2EBW: Allowed characters in keys': { - src: `a!"#$%&'()*+,-./09:;<=>?@AZ[\\]^_\`az{|}~: safe -?foo: safe question mark -:foo: safe colon --foo: safe dash -this is#not: a comment`, - tgt: [ - { - contents: [ - { - items: [ - 'a!"#$%&\'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~', - { node: 'safe' }, - '?foo', - { node: 'safe question mark' }, - ':foo', - { node: 'safe colon' }, - '-foo', - { node: 'safe dash' }, - 'this is#not', - { node: 'a comment' } - ] - } - ] - } - ] - }, - - 'PW8X: Anchors on Empty Scalars': { - src: `- &a -- a -- - &a : a - b: &b - &c : &a -- - ? &d - ? &e - : &a`, - tgt: [ - { - contents: [ - { - items: [ - { node: { anchor: 'a' } }, - { node: 'a' }, - { - node: { - items: [ - { anchor: 'a' }, - { type: Type.MAP_VALUE, node: 'a' }, - 'b', - { type: Type.MAP_VALUE, node: { anchor: 'b' } }, - { anchor: 'c' }, - { type: Type.MAP_VALUE, node: { anchor: 'a' } } - ] - } - }, - { - node: { - items: [ - { type: Type.MAP_KEY, node: { anchor: 'd' } }, - { type: Type.MAP_KEY, node: { anchor: 'e' } }, - { type: Type.MAP_VALUE, node: { anchor: 'a' } } - ] - } - } - ] - } - ] - } - ] - } - } -} - -for (const section in spec) { - describe(section, () => { - for (const name in spec[section]) { - test(name, () => { - const { src, tgt } = spec[section][name] - const documents = parse(src) - trace: 'PARSED', JSON.stringify(pretty(documents), null, ' ') - testSpec(documents, tgt) - const reSrc = String(documents) - trace: 'RE-STRUNG', '\n' + reSrc - // expect(reSrc).toBe(src) - const reDoc = parse(reSrc) - trace: 'RE-PARSED', JSON.stringify(pretty(reDoc), null, ' ') - testSpec(reDoc, tgt) - }) - } - }) -} diff --git a/tests/cst/common.js b/tests/cst/common.js deleted file mode 100644 index 1fb59410..00000000 --- a/tests/cst/common.js +++ /dev/null @@ -1,51 +0,0 @@ -import { Node } from '../../src/cst/Node.js' - -export const pretty = node => { - if (!node || typeof node !== 'object') return node - if (Array.isArray(node)) return node.map(pretty) - const res = {} - if (node.anchor) res.anchor = node.anchor - if (typeof node.tag === 'string') res.tag = node.tag - if (node.comment) res.comment = node.comment - if (node.contents) { - if (node.directives.length > 0) res.directives = node.directives.map(pretty) - if (node.contents.length > 0) res.contents = node.contents.map(pretty) - } else if (node.items) { - res.type = node.type - res.items = node.items.map(pretty) - } else if (typeof node.node !== 'undefined') { - res.type = node.type - res.node = pretty(node.node) - } else if (node.strValue) { - res.strValue = node.strValue - } else if (node.rawValue) { - res.type = node.type - res.rawValue = node.rawValue - } - if (Object.keys(res).every(key => key === 'rawValue')) return res.rawValue - return res -} - -export const testSpec = (res, exp) => { - if (typeof exp === 'string') { - const value = - res instanceof Node - ? 'strValue' in res - ? res.strValue - : res.rawValue - : typeof res === 'string' - ? res - : res && res.char - expect(value).toBe(exp) - } else if (Array.isArray(exp)) { - expect(res).toBeInstanceOf(Array) - trace: 'test-array', exp - exp.forEach((e, i) => testSpec(res[i], e)) - } else if (exp) { - expect(res).toBeInstanceOf(Object) - trace: 'test-object', exp - for (const key in exp) testSpec(res[key], exp[key]) - } else { - expect(res).toBeNull() - } -} diff --git a/tests/cst/corner-cases.js b/tests/cst/corner-cases.js deleted file mode 100644 index c66ebc59..00000000 --- a/tests/cst/corner-cases.js +++ /dev/null @@ -1,585 +0,0 @@ -import { source } from 'common-tags' -import { parse } from '../../src/cst/parse.js' - -describe('folded block with chomp: keep', () => { - test('nl + nl', () => { - const src = `>+\nblock\n\n` - const doc = parse(src)[0] - expect(doc.contents[0].strValue).toBe('block\n\n') - }) - - test('nl + nl + sp + nl', () => { - const src = '>+\nab\n\n \n' - const doc = parse(src)[0] - expect(doc.contents[0].strValue).toBe('ab\n\n \n') - }) -}) - -describe('folded block with indent indicator + leading empty lines + leading whitespace', () => { - test('one blank line', () => { - const src = '>1\n\n line\n' - const doc = parse(src)[0] - expect(doc.contents[0].strValue).toBe('\n line\n') - }) - - test('two blank lines', () => { - const src = '>1\n\n\n line\n' - const doc = parse(src)[0] - expect(doc.contents[0].strValue).toBe('\n\n line\n') - }) -}) - -describe('multiple linebreaks in scalars', () => { - test('plain', () => { - const src = `trimmed\n\n\n\nlines\n` - const doc = parse(src)[0] - expect(doc.contents[0].strValue).toBe('trimmed\n\n\nlines') - }) - - test('single-quoted', () => { - const src = `'trimmed\n\n\n\nlines'\n` - const doc = parse(src)[0] - expect(doc.contents[0].strValue).toBe('trimmed\n\n\nlines') - }) -}) - -test('no null document for document-end marker', () => { - const src = '---\nx\n...\n' - const docs = parse(src) - expect(docs).toHaveLength(1) -}) - -test('explicit key after empty value', () => { - const src = 'one:\n? two\n' - const doc = parse(src)[0] - const raw = doc.contents[0].items.map(it => it.rawValue) - expect(raw).toMatchObject(['one', ':', '? two']) -}) - -test('seq with anchor as explicit key', () => { - const src = '? &key\n- a\n' - const doc = parse(src)[0] - expect(doc.contents).toHaveLength(1) - expect(doc.contents[0].items[0].node.rawValue).toBe('- a') -}) - -test('unindented single-quoted string', () => { - const src = `key: 'two\nlines'\n` - const doc = parse(src)[0] - const { node } = doc.contents[0].items[1] - expect(node.error).toBeNull() - expect(node.strValue).toMatchObject({ - str: 'two lines', - errors: [ - new SyntaxError( - 'Multi-line single-quoted string needs to be sufficiently indented' - ) - ] - }) -}) - -describe('seq unindent to non-empty indent', () => { - test('after map', () => { - // const src = ` - // - a:| - b| - c|` - const src = ` - - a: - - b - - c\n` - const doc = parse(src)[0] - expect(doc.contents).toHaveLength(2) - expect(doc.contents[1].items).toHaveLength(2) - expect(doc.contents[1].items[1].error).toBeNull() - }) - - test('after seq', () => { - const src = ` - - - - a - - b\n` - const doc = parse(src)[0] - expect(doc.contents).toHaveLength(2) - expect(doc.contents[1].items).toHaveLength(2) - expect(doc.contents[1].items[1].error).toBeNull() - }) -}) - -test('eemeli/yaml#10', () => { - const src = ` - a: - - b - c: d -` - const doc = parse(src)[0] - expect(doc.contents).toHaveLength(2) - expect(doc.contents[1].items).toHaveLength(4) - expect(doc.contents[1].items[1].error).toBeNull() -}) - -test('eemeli/yaml#19', () => { - const src = 'a:\n # 123' - const doc = parse(src)[0] - expect(doc.contents).toMatchObject([ - { - type: 'MAP', - items: [ - { type: 'PLAIN', range: { start: 0, end: 1 } }, - { type: 'MAP_VALUE', range: { start: 1, end: 2 } } - ] - }, - { type: 'COMMENT', range: { start: 5, end: 10 } } - ]) -}) - -test('eemeli/yaml#20', () => { - const src = 'a:\r\n 123\r\nb:\r\n 456\r\n' - const docStream = parse(src) - const a = docStream[0].contents[0].items[1].node - expect(a.strValue).toBe('123') - expect(docStream.setOrigRanges()).toBe(true) - const { origStart: a0, origEnd: a1 } = a.valueRange - expect(src.slice(a0, a1)).toBe('123') - const b = docStream[0].contents[0].items[3].node - expect(b.strValue).toBe('456') - const { origStart: b0, origEnd: b1 } = b.valueRange - expect(src.slice(b0, b1)).toBe('456') -}) - -test('eemeli/yaml#38', () => { - const src = ` - - - - - a - - -` - const doc = parse(src)[0] - const { items } = doc.contents[1] - expect(items).toHaveLength(3) - items.forEach(item => { - expect(item.error).toBe(null) - }) -}) - -test('eemeli/yaml#56', () => { - const src = ':,\n' - const doc = parse(src)[0] - expect(doc.contents).toHaveLength(1) - expect(doc.contents[0]).toMatchObject({ - error: null, - strValue: ':,', - type: 'PLAIN' - }) -}) - -describe('collection indicator as last char', () => { - test('seq item', () => { - const src = '-' - const doc = parse(src)[0] - expect(doc.contents[0]).toMatchObject({ - type: 'SEQ', - items: [{ type: 'SEQ_ITEM', node: null }] - }) - }) - - test('explicit map key', () => { - const src = '?' - const doc = parse(src)[0] - expect(doc.contents[0]).toMatchObject({ - type: 'MAP', - items: [{ type: 'MAP_KEY', node: null }] - }) - }) - - test('empty map value', () => { - const src = ':' - const doc = parse(src)[0] - expect(doc.contents[0]).toMatchObject({ - type: 'MAP', - items: [{ type: 'MAP_VALUE', node: null }] - }) - }) - - test('indented seq-in-seq', () => { - const src = ` -\n - - a\n -` - const doc = parse(src)[0] - expect(doc.contents[0]).toMatchObject({ - items: [ - { error: null }, - { node: { items: [{ node: { type: 'PLAIN', strValue: 'a' } }] } }, - { error: null } - ] - }) - }) - - test('implicit map value separator', () => { - const src = 'a:' - const doc = parse(src)[0] - expect(doc.contents[0]).toMatchObject({ - type: 'MAP', - items: [ - { type: 'PLAIN', strValue: 'a' }, - { type: 'MAP_VALUE', node: null } - ] - }) - }) -}) - -test('parse an empty string as an empty document', () => { - const doc = parse('')[0] - expect(doc).toMatchObject({ - error: null, - contents: [] - }) -}) - -test('re-stringify flow seq with comments', () => { - const src = '[ #c\n1, #d\n2 ]\n' - const doc = parse(src) - expect(String(doc)).toBe(src) -}) - -test('blank line after less-indented comment (eemeli/yaml#91)', () => { - const src = ` - foo1: bar -# comment - - foo2: baz` - const doc = parse(src) - expect(doc).toHaveLength(1) - expect(doc[0].contents).toMatchObject([ - { type: 'BLANK_LINE' }, - { type: 'MAP' } - ]) -}) - -describe('flow collection as same-line mapping key value', () => { - test('eemeli/yaml#113', () => { - const src = source` - --- - foo: - bar: - enum: [ - "abc", - "cde" - ] - ` - const doc = parse(src) - const barValue = doc[0].contents[0].items[1].node.items[1].node - expect(barValue.items[1].node).toMatchObject({ - error: null, - items: [ - { char: '[' }, - { type: 'QUOTE_DOUBLE' }, - { char: ',' }, - { type: 'QUOTE_DOUBLE' }, - { char: ']' } - ], - type: 'FLOW_SEQ' - }) - }) - - test('eemeli/yaml#114', () => { - const src = source` - foo: { - bar: boom - } - ` - const doc = parse(src) - const flowCollection = doc[0].contents[0].items[1].node - expect(flowCollection).toMatchObject({ - error: null, - items: [ - { char: '{' }, - { type: 'PLAIN' }, - { char: ':' }, - { type: 'PLAIN' }, - { char: '}' } - ], - type: 'FLOW_MAP' - }) - }) - - test('Fails on insufficient indent', () => { - const src = source` - foo: { - bar: boom - } - ` - const doc = parse(' ' + src) - const flowCollection = doc[0].contents[0].items[1].node - expect(flowCollection).toMatchObject({ - error: { - message: 'Insufficient indentation in flow collection', - name: 'YAMLSemanticError' - }, - items: [ - { char: '{' }, - { type: 'PLAIN' }, - { char: ':' }, - { type: 'PLAIN' }, - { char: '}' } - ], - type: 'FLOW_MAP' - }) - }) -}) - -describe('blank lines before empty collection item value', () => { - test('empty value followed by blank line at document end', () => { - const src = 'a:\n\n' - const doc = parse(src)[0] - expect(doc.contents).toMatchObject([ - { - type: 'MAP', - items: [ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ] - } - ]) - }) - - test('empty value with blank line before comment at document end', () => { - const src = 'a:\n\n#c\n' - const doc = parse(src)[0] - expect(doc.contents).toMatchObject([ - { - type: 'MAP', - items: [ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ] - }, - { type: 'BLANK_LINE', range: { start: 3, end: 4 } }, - { type: 'COMMENT', range: { start: 4, end: 6 } } - ]) - }) - - test('empty value with blank line after inline comment at document end', () => { - const src = 'a: #c\n\n' - const doc = parse(src)[0] - expect(doc.contents).toMatchObject([ - { - type: 'MAP', - items: [ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [{ start: 3, end: 5 }] } - ] - } - ]) - }) - - test('empty value with blank line after separate-line comment at document end', () => { - const src = 'a:\n#c\n\n' - const doc = parse(src)[0] - expect(doc.contents).toMatchObject([ - { - type: 'MAP', - items: [ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ] - }, - { type: 'COMMENT', range: { start: 3, end: 5 } } - ]) - }) - - test('empty value with blank line before & after comment at document end', () => { - const src = 'a:\n\n#c\n\n' - const doc = parse(src)[0] - expect(doc.contents).toMatchObject([ - { - type: 'MAP', - items: [ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ] - }, - { type: 'BLANK_LINE', range: { start: 3, end: 4 } }, - { type: 'COMMENT', range: { start: 4, end: 6 } } - ]) - }) - - test('empty value with blank lines before & after two comments at document end', () => { - const src = 'a:\n\n#c\n\n#d\n\n' - const doc = parse(src)[0] - expect(doc.contents).toMatchObject([ - { - type: 'MAP', - items: [ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ] - }, - { type: 'BLANK_LINE', range: { start: 3, end: 4 } }, - { type: 'COMMENT', range: { start: 4, end: 6 } }, - { type: 'BLANK_LINE', range: { start: 7, end: 8 } }, - { type: 'COMMENT', range: { start: 8, end: 10 } } - ]) - }) - - test('empty value followed by blank line not at end', () => { - const src = 'a:\n\nb:\n' - const doc = parse(src)[0] - expect(doc.contents[0].items).toMatchObject([ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] }, - { type: 'BLANK_LINE', range: { start: 3, end: 4 } }, - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ]) - }) - - test('empty value with blank line before comment not at end', () => { - const src = 'a:\n\n#c\nb:\n' - const doc = parse(src)[0] - expect(doc.contents[0].items).toMatchObject([ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] }, - { type: 'BLANK_LINE', range: { start: 3, end: 4 } }, - { type: 'COMMENT', range: { start: 4, end: 6 } }, - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ]) - }) - - test('empty value with blank line after comment not at end', () => { - const src = 'a: #c\n\nb:\n' - const doc = parse(src)[0] - expect(doc.contents[0].items).toMatchObject([ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [{ start: 3, end: 5 }] }, - { type: 'BLANK_LINE', range: { start: 6, end: 7 } }, - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ]) - }) - - test('empty value with blank line before & after comment not at end', () => { - const src = 'a:\n\n#c\n\nb:\n' - const doc = parse(src)[0] - expect(doc.contents[0].items).toMatchObject([ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] }, - { type: 'BLANK_LINE', range: { start: 3, end: 4 } }, - { type: 'COMMENT', range: { start: 4, end: 6 } }, - { type: 'BLANK_LINE', range: { start: 7, end: 8 } }, - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ]) - }) - - test('empty value with blank line before comment with CR-LF line ends', () => { - const src = '\r\na:\r\n\r\n#c\r\n' - const cst = parse(src) - cst.setOrigRanges() - expect(cst[0].contents).toMatchObject([ - { - type: 'BLANK_LINE', - range: { start: 0, end: 1, origStart: 1, origEnd: 2 } - }, - { - type: 'MAP', - items: [ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ] - }, - { - type: 'BLANK_LINE', - range: { start: 4, end: 5, origStart: 7, origEnd: 8 } - }, - { - type: 'COMMENT', - range: { start: 5, end: 7, origStart: 8, origEnd: 10 } - } - ]) - }) - - test('empty value with blank line after comment with CR-LF line ends', () => { - const src = '\r\na: #c\r\n\r\n' - const cst = parse(src) - cst.setOrigRanges() - expect(cst[0].contents[1].items).toMatchObject([ - { type: 'PLAIN', props: [] }, - { - type: 'MAP_VALUE', - node: null, - props: [{ start: 4, end: 6, origStart: 5, origEnd: 7 }] - } - ]) - }) - - test('empty value with blank line before & after comment with CR-LF line ends', () => { - const src = '\r\na:\r\n\r\n#c\r\n\r\nb:\r\n' - const cst = parse(src) - cst.setOrigRanges() - expect(cst[0].contents).toMatchObject([ - { - type: 'BLANK_LINE', - range: { start: 0, end: 1, origStart: 1, origEnd: 2 } - }, - { - type: 'MAP', - items: [ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] }, - { - type: 'BLANK_LINE', - range: { start: 4, end: 5, origStart: 7, origEnd: 8 } - }, - { - type: 'COMMENT', - range: { start: 5, end: 7, origStart: 8, origEnd: 10 } - }, - { - type: 'BLANK_LINE', - range: { start: 8, end: 9, origStart: 13, origEnd: 14 } - }, - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ] - } - ]) - }) - - test('empty value with blank lines before & after two comments with CR-LF line ends', () => { - const src = '\r\na:\r\n\r\n#c\r\n\r\n#d\r\n\r\nb:\r\n' - const cst = parse(src) - cst.setOrigRanges() - expect(cst[0].contents).toMatchObject([ - { - type: 'BLANK_LINE', - range: { start: 0, end: 1, origStart: 1, origEnd: 2 } - }, - { - type: 'MAP', - items: [ - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] }, - { - type: 'BLANK_LINE', - range: { start: 4, end: 5, origStart: 7, origEnd: 8 } - }, - { - type: 'COMMENT', - range: { start: 5, end: 7, origStart: 8, origEnd: 10 } - }, - { - type: 'BLANK_LINE', - range: { start: 8, end: 9, origStart: 13, origEnd: 14 } - }, - { - type: 'COMMENT', - range: { start: 9, end: 11, origStart: 14, origEnd: 16 } - }, - { - type: 'BLANK_LINE', - range: { start: 12, end: 13, origStart: 19, origEnd: 20 } - }, - { type: 'PLAIN', props: [] }, - { type: 'MAP_VALUE', node: null, props: [] } - ] - } - ]) - }) -}) diff --git a/tests/cst/parse.js b/tests/cst/parse.js deleted file mode 100644 index 0e2a4db4..00000000 --- a/tests/cst/parse.js +++ /dev/null @@ -1,307 +0,0 @@ -import { parse } from '../../src/cst/parse.js' - -test('return value', () => { - const src = '---\n- foo\n- bar\n' - const cst = parse(src) - expect(cst).toHaveLength(1) - expect(cst[0]).toMatchObject({ - contents: [ - { - error: null, - items: [ - { - error: null, - node: { - error: null, - props: [], - range: { end: 9, start: 6 }, - type: 'PLAIN', - value: null, - valueRange: { end: 9, start: 6 } - }, - props: [], - range: { end: 9, start: 4 }, - type: 'SEQ_ITEM', - value: null, - valueRange: { end: 9, start: 4 } - }, - { - error: null, - node: { - error: null, - props: [], - range: { end: 15, start: 12 }, - type: 'PLAIN', - value: null, - valueRange: { end: 15, start: 12 } - }, - props: [], - range: { end: 15, start: 10 }, - type: 'SEQ_ITEM', - value: null, - valueRange: { end: 15, start: 10 } - } - ], - props: [], - range: { end: 16, start: 4 }, - type: 'SEQ', - value: null, - valueRange: { end: 15, start: 4 } - } - ], - directives: [], - directivesEndMarker: { start: 0, end: 3 }, - documentEndMarker: null, - error: null, - props: [], - range: null, - type: 'DOCUMENT', - value: null, - valueRange: { start: 3, end: 16 } - }) -}) - -describe('toString()', () => { - test('plain document', () => { - const src = '- foo\n- bar\n' - const cst = parse(src) - expect(String(cst)).toBe(src) - }) - - test('stream of two documents', () => { - const src = 'foo\n...\nbar\n' - const cst = parse(src) - expect(cst).toHaveLength(2) - expect(String(cst)).toBe(src) - }) - - test('document with CRLF line separators', () => { - const src = '- foo\r\n- bar\r\n' - const cst = parse(src) - expect(cst.toString()).toBe('- foo\n- bar\n') - }) -}) - -describe('setOrigRanges()', () => { - test('return false for no CRLF', () => { - const src = '- foo\n- bar\n' - const cst = parse(src) - expect(cst.setOrigRanges()).toBe(false) - expect(cst[0].valueRange).toMatchObject({ start: 0, end: 12 }) - expect(cst[0].valueRange.origStart).toBeUndefined() - expect(cst[0].valueRange.origEnd).toBeUndefined() - }) - - test('no error on comments', () => { - const src = '\r\n# hello' - expect(() => parse(src).setOrigRanges()).not.toThrowError() - }) - - test('directives', () => { - const src = '\r\n%YAML 1.2\r\n---\r\nfoo\r\n' - const cst = parse(src) - expect(cst.setOrigRanges()).toBe(true) - expect(cst[0]).toMatchObject({ - directives: [ - { type: 'BLANK_LINE' }, - { - name: 'YAML', - range: { end: 10, origEnd: 11, origStart: 2, start: 1 } - } - ], - contents: [ - { - type: 'PLAIN', - range: { end: 18, origEnd: 21, origStart: 18, start: 15 } - } - ] - }) - }) - - test('block scalar', () => { - const src = '|\r\n foo\r\n bar\r\n' - const cst = parse(src) - expect(cst.setOrigRanges()).toBe(true) - const node = cst[0].contents[0] - expect(node).toMatchObject({ - type: 'BLOCK_LITERAL', - range: { end: 14, origEnd: 17, origStart: 0, start: 0 } - }) - expect(node.strValue).toBe('foo\nbar\n') - }) - - test('single document', () => { - const src = '- foo\r\n- bar\r\n' - const cst = parse(src) - expect(cst.setOrigRanges()).toBe(true) - expect(cst).toHaveLength(1) - const { range, valueRange } = cst[0].contents[0].items[1].node - expect(src.slice(range.origStart, range.origEnd)).toBe('bar') - expect(src.slice(valueRange.origStart, valueRange.origEnd)).toBe('bar') - expect(cst[0]).toMatchObject({ - contents: [ - { - error: null, - items: [ - { - error: null, - node: { - error: null, - props: [], - range: { end: 5, origEnd: 5, origStart: 2, start: 2 }, - type: 'PLAIN', - value: null, - valueRange: { end: 5, origEnd: 5, origStart: 2, start: 2 } - }, - props: [], - range: { end: 5, origEnd: 5, origStart: 0, start: 0 }, - type: 'SEQ_ITEM', - value: null, - valueRange: { end: 5, origEnd: 5, origStart: 0, start: 0 } - }, - { - error: null, - node: { - error: null, - props: [], - range: { end: 11, origEnd: 12, origStart: 9, start: 8 }, - type: 'PLAIN', - value: null, - valueRange: { end: 11, origEnd: 12, origStart: 9, start: 8 } - }, - props: [], - range: { end: 11, origEnd: 12, origStart: 7, start: 6 }, - type: 'SEQ_ITEM', - value: null, - valueRange: { end: 11, origEnd: 12, origStart: 7, start: 6 } - } - ], - props: [], - range: { end: 12, origEnd: 14, origStart: 0, start: 0 }, - type: 'SEQ', - value: null, - valueRange: { end: 11, origEnd: 12, origStart: 0, start: 0 } - } - ], - directives: [], - directivesEndMarker: null, - documentEndMarker: null, - error: null, - props: [], - range: null, - type: 'DOCUMENT', - value: null, - valueRange: { end: 12, origEnd: 14, origStart: 0, start: 0 } - }) - expect(cst[0].context.root).toBe(cst[0]) - expect(cst[0].contents[0].items[1].node.context.root).toBe(cst[0]) - }) - - test('stream of two documents', () => { - const src = 'foo\r\n...\r\nbar\r\n' - const cst = parse(src) - expect(cst.setOrigRanges()).toBe(true) - expect(cst).toHaveLength(2) - const { range, valueRange } = cst[1].contents[0] - expect(src.slice(range.origStart, range.origEnd)).toBe('bar') - expect(src.slice(valueRange.origStart, valueRange.origEnd)).toBe('bar') - expect(cst[0]).toMatchObject({ - contents: [ - { - error: null, - props: [], - range: { end: 3, origEnd: 3, origStart: 0, start: 0 }, - type: 'PLAIN', - value: null, - valueRange: { end: 3, origEnd: 3, origStart: 0, start: 0 } - } - ], - directives: [], - directivesEndMarker: null, - documentEndMarker: { start: 4, end: 7, origStart: 5, origEnd: 8 }, - error: null, - props: [], - range: null, - type: 'DOCUMENT', - value: null, - valueRange: { end: 4, origEnd: 5, origStart: 0, start: 0 } - }) - expect(cst[1]).toMatchObject({ - contents: [ - { - error: null, - props: [], - range: { end: 11, origEnd: 13, origStart: 10, start: 8 }, - type: 'PLAIN', - value: null, - valueRange: { end: 11, origEnd: 13, origStart: 10, start: 8 } - } - ], - directives: [], - directivesEndMarker: null, - documentEndMarker: null, - error: null, - props: [], - range: null, - type: 'DOCUMENT', - value: null, - valueRange: { end: 12, origEnd: 15, origStart: 10, start: 8 } - }) - expect(cst[0].context.root).toBe(cst[0]) - expect(cst[1].context.root).toBe(cst[1]) - }) - - test('flow collections', () => { - const src = '\r\n{ : }\r\n' - const cst = parse(src) - expect(() => cst.setOrigRanges()).not.toThrowError() - expect(cst[0]).toMatchObject({ - contents: [ - { - error: null, - props: [], - range: { end: 1, origEnd: 2, origStart: 1, start: 0 }, - type: 'BLANK_LINE', - value: null, - valueRange: null - }, - { - error: null, - items: [ - { char: '{', offset: 1, origOffset: 2 }, - { char: ':', offset: 3, origOffset: 4 }, - { char: '}', offset: 5, origOffset: 6 } - ], - props: [], - range: { end: 6, origEnd: 7, origStart: 2, start: 1 }, - type: 'FLOW_MAP', - value: null, - valueRange: { end: 6, origEnd: 7, origStart: 2, start: 1 } - } - ], - directives: [], - directivesEndMarker: null, - documentEndMarker: null, - error: null, - props: [], - range: null, - type: 'DOCUMENT', - value: null, - valueRange: { end: 7, origEnd: 9, origStart: 2, start: 1 } - }) - expect(cst[0].context.root).toBe(cst[0]) - expect(cst[0].contents[1].context.root).toBe(cst[0]) - }) -}) - -test('blank lines', () => { - const src = '#cc\n\n\n\n##dd' - const cst = parse(src) - expect(cst[0].contents).toMatchObject([ - { type: 'COMMENT', error: null }, - { type: 'BLANK_LINE', error: null }, - { type: 'BLANK_LINE', error: null }, - { type: 'BLANK_LINE', error: null }, - { type: 'COMMENT', error: null } - ]) -}) diff --git a/tests/cst/set-value.js b/tests/cst/set-value.js deleted file mode 100644 index ba5e8e70..00000000 --- a/tests/cst/set-value.js +++ /dev/null @@ -1,53 +0,0 @@ -import { Type } from '../../src/constants.js' -import { parse } from '../../src/cst/parse.js' -import { CollectionItem } from '../../src/cst/CollectionItem.js' - -test('set value in collection', () => { - const src = `- Mark McGwire -- Sammy Sosa -- Ken Griffey -` // spec 2.1 - const cst = parse(src) - cst[0].contents[0].items[1].node.value = 'TEST\n' - expect(String(cst)).toBe(src.replace(/Sammy Sosa/, 'TEST')) -}) - -test('replace entire contents', () => { - const src = `- Mark McGwire -- Sammy Sosa -- Ken Griffey -` // spec 2.1 - const cst = parse(src) - cst[0].contents[0].value = 'TEST: true\n' - expect(String(cst)).toBe('TEST: true\n') -}) - -test('remove map key/value pair', () => { - const src = `hr: 65 # Home runs -avg: 0.278 # Batting average -rbi: 147 # Runs Batted In -` // spec 2.2 - const cst = parse(src) - cst[0].contents[0].items[2].value = '' - cst[0].contents[0].items[3].value = '' - expect(String(cst)).toBe(src.replace(/avg.*\n/, '')) -}) - -test('add entry to seq', () => { - const src = `american: - - Boston Red Sox - - Detroit Tigers - - New York Yankees -national: - - New York Mets - - Chicago Cubs - - Atlanta Braves -` // spec 2.3 - const cst = parse(src) - const seq = cst[0].contents[0].items[3].node - const item = new CollectionItem(Type.SEQ_ITEM) - item.context = seq.items[2].context - item.value = '- "TEST"\n' - seq.items.push(item) - expect(String(cst)).toBe(`${src} ${item.value}`) -}) diff --git a/tests/cst/source-utils.js b/tests/cst/source-utils.js deleted file mode 100644 index 489b6e17..00000000 --- a/tests/cst/source-utils.js +++ /dev/null @@ -1,49 +0,0 @@ -import { getLinePos } from '../../src/cst/source-utils.js' -import { parse } from '../../src/cst/parse.js' - -test('lineStarts for empty document', () => { - const src = '' - const cst = parse(src) - expect(() => getLinePos(0, cst)).not.toThrow() - expect(cst[0].lineStarts).toMatchObject([0]) -}) - -test('lineStarts for multiple documents', () => { - const src = 'foo\n...\nbar\n' - const cst = parse(src) - expect(() => getLinePos(0, cst)).not.toThrow() - expect(cst[0].lineStarts).toMatchObject([0, 4, 8, 12]) -}) - -test('lineStarts for malformed document', () => { - const src = '- foo\n\t- bar\n' - const cst = parse(src) - expect(() => getLinePos(0, cst)).not.toThrow() - expect(cst[0].lineStarts).toMatchObject([0, 6, 13]) -}) - -test('getLinePos()', () => { - const src = '- foo\n- bar\n' - const cst = parse(src) - expect(cst[0].lineStarts).toBeUndefined() - expect(getLinePos(0, cst)).toMatchObject({ line: 1, col: 1 }) - expect(getLinePos(1, cst)).toMatchObject({ line: 1, col: 2 }) - expect(getLinePos(2, cst)).toMatchObject({ line: 1, col: 3 }) - expect(getLinePos(5, cst)).toMatchObject({ line: 1, col: 6 }) - expect(getLinePos(6, cst)).toMatchObject({ line: 2, col: 1 }) - expect(getLinePos(7, cst)).toMatchObject({ line: 2, col: 2 }) - expect(getLinePos(11, cst)).toMatchObject({ line: 2, col: 6 }) - expect(getLinePos(12, cst)).toMatchObject({ line: 3, col: 1 }) - expect(cst[0].lineStarts).toMatchObject([0, 6, 12]) -}) - -test('invalid args for getLinePos()', () => { - const src = '- foo\n- bar\n' - const cst = parse(src) - expect(getLinePos()).toBeNull() - expect(getLinePos(0)).toBeNull() - expect(getLinePos(1)).toBeNull() - expect(getLinePos(-1, cst)).toBeNull() - expect(getLinePos(13, cst)).toBeNull() - expect(getLinePos(Math.MAXINT, cst)).toBeNull() -}) diff --git a/tests/doc/comments.js b/tests/doc/comments.js index dc0bc785..00b2f958 100644 --- a/tests/doc/comments.js +++ b/tests/doc/comments.js @@ -683,19 +683,6 @@ entryB: expect(String(doc)).toBe(`a: b #c\n\n#d\n`) }) - test.skip('comment association by indentation', () => { - const src = ` -a: - - b #c -#d\n` - const cst = YAML.parseCST(src) - const collection = cst[0].contents[1] - expect(collection.items).toHaveLength(2) - const comment = cst[0].contents[2] - expect(comment.type).toBe('COMMENT') - expect(comment.comment).toBe('d') - }) - test('blank line after seq in map', () => { const src = `a: - aa diff --git a/tests/doc/errors.js b/tests/doc/errors.js index 4f136dd4..ef302d9f 100644 --- a/tests/doc/errors.js +++ b/tests/doc/errors.js @@ -1,4 +1,3 @@ -import { Node } from '../../src/cst/Node.js' import { YAMLError } from '../../src/errors.js' import * as YAML from '../../src/index.js' @@ -7,8 +6,6 @@ test('require a message and source for all errors', () => { expect(() => new YAMLError()).toThrow(exp) expect(() => new YAMLError('Foo')).toThrow(exp) expect(() => new YAMLError('Foo', {})).toThrow(exp) - expect(() => new YAMLError('Foo', new Node())).toThrow(exp) - expect(() => new YAMLError('Foo', new Node(), 'foo')).not.toThrow() }) test('fail on map value indented with tab', () => { @@ -37,12 +34,12 @@ describe.skip('eemeli/yaml#7', () => { { name: 'YAMLParseError', offset: 16 }, { name: 'YAMLParseError', offset: 17 } ]) - const node = docs[0].errors[0].source - expect(node).toBeInstanceOf(Node) - expect(node.rangeAsLinePos).toMatchObject({ - start: { line: 1, col: 1 }, - end: { line: 1, col: 6 } - }) + // const node = docs[0].errors[0].source + // expect(node).toBeInstanceOf(Node) + // expect(node.rangeAsLinePos).toMatchObject({ + // start: { line: 1, col: 1 }, + // end: { line: 1, col: 6 } + // }) }) test('seq', () => { const src = '[ , ]\n---\n[ 123,,, ]\n' @@ -54,12 +51,12 @@ describe.skip('eemeli/yaml#7', () => { { name: 'YAMLParseError', offset: 16 }, { name: 'YAMLParseError', offset: 17 } ]) - const node = docs[1].errors[0].source - expect(node).toBeInstanceOf(Node) - expect(node.rangeAsLinePos).toMatchObject({ - start: { line: 3, col: 1 }, - end: { line: 3, col: 11 } - }) + // const node = docs[1].errors[0].source + // expect(node).toBeInstanceOf(Node) + // expect(node.rangeAsLinePos).toMatchObject({ + // start: { line: 3, col: 1 }, + // end: { line: 3, col: 11 } + // }) }) }) diff --git a/types.d.ts b/types.d.ts index b47aea93..c6a71d2c 100644 --- a/types.d.ts +++ b/types.d.ts @@ -1,5 +1,4 @@ import { Document, scalarOptions } from './index' -import { CST } from './cst' import { Type } from './util' export const binaryOptions: scalarOptions.Binary @@ -179,7 +178,7 @@ export class Node { /** A comment before this */ commentBefore?: string | null /** Only available when `keepCstNodes` is set to `true` */ - cstNode?: CST.Node + // cstNode?: CST.Node /** * The [start, end] range of characters of the source parsed * into this node (undefined for pairs or if not parsed) @@ -219,7 +218,7 @@ export namespace Scalar { export class Alias extends Node { type: Type.ALIAS source: Node - cstNode?: CST.Alias + // cstNode?: CST.Alias toString(ctx: Schema.StringifyContext): string } @@ -230,7 +229,7 @@ export class Pair extends Node { key: any /** Always Node or null when parsed, but can be set to anything. */ value: any - cstNode?: never // no corresponding cstNode + // cstNode?: never // no corresponding cstNode toJSON(arg?: any, ctx?: AST.NodeToJsonContext): object | Map toString( ctx?: Schema.StringifyContext, @@ -332,48 +331,48 @@ export namespace AST { interface BlockFolded extends Scalar { type: Type.BLOCK_FOLDED - cstNode?: CST.BlockFolded + // cstNode?: CST.BlockFolded } interface BlockLiteral extends Scalar { type: Type.BLOCK_LITERAL - cstNode?: CST.BlockLiteral + // cstNode?: CST.BlockLiteral } interface PlainValue extends Scalar { type: Type.PLAIN - cstNode?: CST.PlainValue + // cstNode?: CST.PlainValue } interface QuoteDouble extends Scalar { type: Type.QUOTE_DOUBLE - cstNode?: CST.QuoteDouble + // cstNode?: CST.QuoteDouble } interface QuoteSingle extends Scalar { type: Type.QUOTE_SINGLE - cstNode?: CST.QuoteSingle + // cstNode?: CST.QuoteSingle } interface FlowMap extends YAMLMap { type: Type.FLOW_MAP - cstNode?: CST.FlowMap + // cstNode?: CST.FlowMap } interface BlockMap extends YAMLMap { type: Type.MAP - cstNode?: CST.Map + // cstNode?: CST.Map } interface FlowSeq extends YAMLSeq { type: Type.FLOW_SEQ items: Array - cstNode?: CST.FlowSeq + // cstNode?: CST.FlowSeq } interface BlockSeq extends YAMLSeq { type: Type.SEQ items: Array - cstNode?: CST.Seq + // cstNode?: CST.Seq } } diff --git a/util.d.ts b/util.d.ts index 50d03b15..dacd5b84 100644 --- a/util.d.ts +++ b/util.d.ts @@ -1,4 +1,3 @@ -import { CST } from './cst' import { Pair, Scalar, Schema } from './types' export function findPair(items: any[], key: Scalar | any): Pair | undefined @@ -45,11 +44,11 @@ export class YAMLError extends Error { | 'YAMLSyntaxError' | 'YAMLWarning' message: string - source?: CST.Node + offset?: number - nodeType?: Type - range?: CST.Range - linePos?: { start: LinePos; end: LinePos } + // nodeType?: Type + // range?: CST.Range + // linePos?: { start: LinePos; end: LinePos } /** * Drops `source` and adds `nodeType`, `range` and `linePos`, as well as From beb8d8b3eae8509009b276ae7a99a13b0574c410 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 7 Feb 2021 10:28:53 +0200 Subject: [PATCH 80/89] Drop YAMLReferenceError, YAMLSemanticError & YAMLSyntaxError in favour of YAMLParseError --- docs/06_custom_tags.md | 2 +- docs/08_errors.md | 24 ++++++++---------------- src/ast/Alias.js | 7 ++----- src/errors.d.ts | 22 +--------------------- src/errors.js | 18 ------------------ src/index.ts | 4 ++-- src/util.js | 8 +------- util.d.ts | 18 +++--------------- util.js | 4 +--- util.mjs | 4 +--- 10 files changed, 20 insertions(+), 91 deletions(-) diff --git a/docs/06_custom_tags.md b/docs/06_custom_tags.md index c198784c..5b653941 100644 --- a/docs/06_custom_tags.md +++ b/docs/06_custom_tags.md @@ -132,7 +132,7 @@ import { stringifyString, // (node, ctx, ...) => string toJS, // (value, arg, ctx) => any -- Recursively convert to plain JS Type, // { [string]: string } -- Used as enum for node types - YAMLReferenceError, YAMLSemanticError, YAMLSyntaxError, YAMLWarning + YAMLError, YAMLParseError, YAMLWarning } from 'yaml/util' ``` diff --git a/docs/08_errors.md b/docs/08_errors.md index 4ef8c1a0..dfbd98f6 100644 --- a/docs/08_errors.md +++ b/docs/08_errors.md @@ -2,13 +2,13 @@ Nearly all errors and warnings produced by the `yaml` parser functions contain the following fields: -| Member | Type | Description | -| ------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| name | `string` | One of `YAMLReferenceError`, `YAMLSemanticError`, `YAMLSyntaxError`, or `YAMLWarning` | -| message | `string` | A human-readable description of the error | -| source | `CST Node` | The CST node at which this error or warning was encountered. Note that in particular `source.context` is likely to be a complex object and include some circular references. | +| Member | Type | Description | +| ------- | -------- | ------------------------------------------------------------------------ | +| name | `string` | Either `YAMLParseError` or `YAMLWarning` | +| message | `string` | A human-readable description of the error | +| offset | `number` | The offset in the source at which this error or warning was encountered. | -If the `prettyErrors` option is enabled, `source` is dropped from the errors and the following fields are added with summary information regarding the error's source node, if available: +If the `prettyErrors` option is enabled, the following fields are added with summary information regarding the error's source node, if available: | Member | Type | Description | | -------- | ----------------------------------- | --------------------------------------------------------------------------------------------- | @@ -18,17 +18,9 @@ If the `prettyErrors` option is enabled, `source` is dropped from the errors and In rare cases, the library may produce a more generic error. In particular, `TypeError` may occur when parsing invalid input using the `json` schema, and `ReferenceError` when the `maxAliasCount` limit is enountered. -## YAMLReferenceError +## YAMLParseError -An error resolving a tag or an anchor that is referred to in the source. It is likely that the contents of the `source` node have not been completely parsed into the document. Not used by the CST parser. - -## YAMLSemanticError - -An error related to the metadata of the document, or an error with limitations imposed by the YAML spec. The data contents of the document should be valid, but the metadata may be broken. - -## YAMLSyntaxError - -A serious parsing error; the document contents will not be complete, and the CST is likely to be rather broken. +An error encountered while parsing a source as YAML. ## YAMLWarning diff --git a/src/ast/Alias.js b/src/ast/Alias.js index 4d1b9242..1d6958d0 100644 --- a/src/ast/Alias.js +++ b/src/ast/Alias.js @@ -1,5 +1,4 @@ import { Type } from '../constants.js' -import { YAMLReferenceError } from '../errors.js' import { Collection } from './Collection.js' import { Node } from './Node.js' import { Pair } from './Pair.js' @@ -58,8 +57,7 @@ export class Alias extends Node { /* istanbul ignore if */ if (!anchor || anchor.res === undefined) { const msg = 'This should not happen: Alias anchor was not resolved?' - if (this.cstNode) throw new YAMLReferenceError(this.cstNode, msg) - else throw new ReferenceError(msg) + throw new ReferenceError(msg) } if (maxAliasCount >= 0) { anchor.count += 1 @@ -68,8 +66,7 @@ export class Alias extends Node { if (anchor.count * anchor.aliasCount > maxAliasCount) { const msg = 'Excessive alias count indicates a resource exhaustion attack' - if (this.cstNode) throw new YAMLReferenceError(this.cstNode, msg) - else throw new ReferenceError(msg) + throw new ReferenceError(msg) } } return anchor.res diff --git a/src/errors.d.ts b/src/errors.d.ts index 1c820c43..b147db6c 100644 --- a/src/errors.d.ts +++ b/src/errors.d.ts @@ -6,12 +6,7 @@ interface LinePos { } export class YAMLError extends Error { - name: - | 'YAMLParseError' - | 'YAMLReferenceError' - | 'YAMLSemanticError' - | 'YAMLSyntaxError' - | 'YAMLWarning' + name: 'YAMLParseError' | 'YAMLWarning' message: string offset?: number @@ -32,21 +27,6 @@ export class YAMLParseError extends YAMLError { constructor(source: number, message: string) } -export class YAMLReferenceError extends YAMLError { - name: 'YAMLReferenceError' - constructor(source: number, message: string) -} - -export class YAMLSemanticError extends YAMLError { - name: 'YAMLSemanticError' - constructor(source: number, message: string) -} - -export class YAMLSyntaxError extends YAMLError { - name: 'YAMLSyntaxError' - constructor(source: number, message: string) -} - export class YAMLWarning extends YAMLError { name: 'YAMLWarning' constructor(source: number, message: string) diff --git a/src/errors.js b/src/errors.js index 2fb44ecc..06339457 100644 --- a/src/errors.js +++ b/src/errors.js @@ -39,24 +39,6 @@ export class YAMLParseError extends YAMLError { } } -export class YAMLReferenceError extends YAMLError { - constructor(source, message) { - super('YAMLReferenceError', source, message) - } -} - -export class YAMLSemanticError extends YAMLError { - constructor(source, message) { - super('YAMLSemanticError', source, message) - } -} - -export class YAMLSyntaxError extends YAMLError { - constructor(source, message) { - super('YAMLSyntaxError', source, message) - } -} - export class YAMLWarning extends YAMLError { constructor(source, message) { super('YAMLWarning', source, message) diff --git a/src/index.ts b/src/index.ts index e9036a76..02b51cbe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ import { composeStream } from './compose/compose-stream.js' import { LogLevel } from './constants.js' import { Document } from './doc/Document.js' -import { YAMLSemanticError } from './errors.js' +import { YAMLParseError } from './errors.js' import { warn } from './log.js' import { Options } from './options.js' @@ -23,7 +23,7 @@ export function parseDocument(src: string, options?: Options) { ) { const errMsg = 'Source contains multiple documents; please use YAML.parseAllDocuments()' - doc.errors.push(new YAMLSemanticError(docs[1].range[0], errMsg)) + doc.errors.push(new YAMLParseError(docs[1].range[0], errMsg)) } return doc } diff --git a/src/util.js b/src/util.js index 71aab825..a2773d0d 100644 --- a/src/util.js +++ b/src/util.js @@ -4,10 +4,4 @@ export { stringifyNumber } from './stringify/stringifyNumber.js' export { stringifyString } from './stringify/stringifyString.js' export { Type } from './constants.js' -export { - YAMLError, - YAMLReferenceError, - YAMLSemanticError, - YAMLSyntaxError, - YAMLWarning -} from './errors.js' +export { YAMLError, YAMLParseError, YAMLWarning } from './errors.js' diff --git a/util.d.ts b/util.d.ts index dacd5b84..23fc3948 100644 --- a/util.d.ts +++ b/util.d.ts @@ -38,11 +38,7 @@ interface LinePos { } export class YAMLError extends Error { - name: - | 'YAMLReferenceError' - | 'YAMLSemanticError' - | 'YAMLSyntaxError' - | 'YAMLWarning' + name: 'YAMLParseError' | 'YAMLWarning' message: string offset?: number @@ -58,16 +54,8 @@ export class YAMLError extends Error { makePretty(): void } -export class YAMLReferenceError extends YAMLError { - name: 'YAMLReferenceError' -} - -export class YAMLSemanticError extends YAMLError { - name: 'YAMLSemanticError' -} - -export class YAMLSyntaxError extends YAMLError { - name: 'YAMLSyntaxError' +export class YAMLParseError extends YAMLError { + name: 'YAMLParseError' } export class YAMLWarning extends YAMLError { diff --git a/util.js b/util.js index d9472bcb..fd4bf265 100644 --- a/util.js +++ b/util.js @@ -8,7 +8,5 @@ exports.stringifyString = util.stringifyString exports.Type = util.Type exports.YAMLError = util.YAMLError -exports.YAMLReferenceError = util.YAMLReferenceError -exports.YAMLSemanticError = util.YAMLSemanticError -exports.YAMLSyntaxError = util.YAMLSyntaxError +exports.YAMLParseError = util.YAMLParseError exports.YAMLWarning = util.YAMLWarning diff --git a/util.mjs b/util.mjs index 9df8eaa7..c4a04772 100644 --- a/util.mjs +++ b/util.mjs @@ -9,7 +9,5 @@ export const stringifyString = util.stringifyString export const Type = util.Type export const YAMLError = util.YAMLError -export const YAMLReferenceError = util.YAMLReferenceError -export const YAMLSemanticError = util.YAMLSemanticError -export const YAMLSyntaxError = util.YAMLSyntaxError +export const YAMLParseError = util.YAMLParseError export const YAMLWarning = util.YAMLWarning From 0e7ea1c1d5a9091c05e7b7b7d3918c41c2a1cda0 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 7 Feb 2021 10:38:44 +0200 Subject: [PATCH 81/89] Refactor errors.js as TypeScript --- rollup.dev-config.js | 9 ++------- src/errors.d.ts | 33 --------------------------------- src/{errors.js => errors.ts} | 28 +++++++++++++++++++++------- 3 files changed, 23 insertions(+), 47 deletions(-) delete mode 100644 src/errors.d.ts rename src/{errors.js => errors.ts} (61%) diff --git a/rollup.dev-config.js b/rollup.dev-config.js index 651229cf..cb3b1f20 100644 --- a/rollup.dev-config.js +++ b/rollup.dev-config.js @@ -2,13 +2,8 @@ import { resolve } from 'path' import babel from '@rollup/plugin-babel' export default { - input: [ - 'src/ast/index.js', - 'src/doc/Document.js', - 'src/errors.js', - 'src/options.js' - ], - external: [resolve('src/doc/directives.js')], + input: ['src/ast/index.js', 'src/doc/Document.js', 'src/options.js'], + external: [resolve('src/doc/directives.js'), resolve('src/errors.js')], output: { dir: 'lib', format: 'cjs', esModule: false, preserveModules: true }, plugins: [babel()] } diff --git a/src/errors.d.ts b/src/errors.d.ts deleted file mode 100644 index b147db6c..00000000 --- a/src/errors.d.ts +++ /dev/null @@ -1,33 +0,0 @@ -// import { Type } from './constants' - -interface LinePos { - line: number - col: number -} - -export class YAMLError extends Error { - name: 'YAMLParseError' | 'YAMLWarning' - message: string - offset?: number - - // nodeType?: Type - // range?: CST.Range - // linePos?: { start: LinePos; end: LinePos } - - /** - * Drops `source` and adds `nodeType`, `range` and `linePos`, as well as - * adding details to `message`. Run automatically for document errors if - * the `prettyErrors` option is set. - */ - makePretty(): void -} - -export class YAMLParseError extends YAMLError { - name: 'YAMLParseError' - constructor(source: number, message: string) -} - -export class YAMLWarning extends YAMLError { - name: 'YAMLWarning' - constructor(source: number, message: string) -} diff --git a/src/errors.js b/src/errors.ts similarity index 61% rename from src/errors.js rename to src/errors.ts index 06339457..1bf87d45 100644 --- a/src/errors.js +++ b/src/errors.ts @@ -1,5 +1,16 @@ +// import { Type } from './constants' +// interface LinePos { line: number; col: number } + export class YAMLError extends Error { - constructor(name, offset, message) { + name: 'YAMLParseError' | 'YAMLWarning' + message: string + offset?: number + + // nodeType?: Type + // range?: CST.Range + // linePos?: { start: LinePos; end: LinePos } + + constructor(name: YAMLError['name'], offset: number | null, message: string) { if (!message) throw new Error(`Invalid arguments for new ${name}`) super() this.name = name @@ -7,8 +18,12 @@ export class YAMLError extends Error { if (typeof offset === 'number') this.offset = offset } + /** + * Drops `source` and adds `nodeType`, `range` and `linePos`, as well as + * adding details to `message`. Run automatically for document errors if + * the `prettyErrors` option is set. + */ makePretty() { - if (!this.source) return // this.nodeType = this.source.type // const cst = this.source.context && this.source.context.root // if (typeof this.offset === 'number') { @@ -29,18 +44,17 @@ export class YAMLError extends Error { // const ctx = cst && getPrettyContext(this.linePos, cst) // if (ctx) this.message += `:\n\n${ctx}\n` // } - delete this.source } } export class YAMLParseError extends YAMLError { - constructor(source, message) { - super('YAMLParseError', source, message) + constructor(offset: number | null, message: string) { + super('YAMLParseError', offset, message) } } export class YAMLWarning extends YAMLError { - constructor(source, message) { - super('YAMLWarning', source, message) + constructor(offset: number | null, message: string) { + super('YAMLWarning', offset, message) } } From 5bc1066ebebc1470483f6f9cd6fbb06ed06d9d00 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 8 Feb 2021 14:43:06 +0200 Subject: [PATCH 82/89] Refactor composeStream function into Composer class --- src/compose/compose-stream.ts | 191 ---------------------------------- src/compose/composer.ts | 191 ++++++++++++++++++++++++++++++++++ src/index.ts | 60 ++++++++--- 3 files changed, 236 insertions(+), 206 deletions(-) delete mode 100644 src/compose/compose-stream.ts create mode 100644 src/compose/composer.ts diff --git a/src/compose/compose-stream.ts b/src/compose/compose-stream.ts deleted file mode 100644 index a52c562d..00000000 --- a/src/compose/compose-stream.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { Collection, Node } from '../ast/index.js' -import { Directives } from '../doc/directives.js' -import { Document } from '../doc/Document.js' -import { YAMLParseError, YAMLWarning } from '../errors.js' -import { defaultOptions, Options } from '../options.js' -import { CSTParser } from '../parse/cst-parser.js' -import { composeDoc } from './compose-doc.js' -import { resolveEnd } from './resolve-end.js' - -function parsePrelude(prelude: string[]) { - let comment = '' - let atComment = false - let afterEmptyLine = false - for (let i = 0; i < prelude.length; ++i) { - const source = prelude[i] - switch (source[0]) { - case '#': - comment += - (comment === '' ? '' : afterEmptyLine ? '\n\n' : '\n') + - source.substring(1) - atComment = true - afterEmptyLine = false - break - case '%': - if (prelude[i + 1][0] !== '#') i += 1 - atComment = false - break - default: - // This may be wrong after doc-end, but in that case it doesn't matter - if (!atComment) afterEmptyLine = true - atComment = false - } - } - return { comment, afterEmptyLine } -} - -export interface EmptyStream extends Array { - empty: true - comment: string - directives: Directives - errors: YAMLParseError[] - warnings: YAMLWarning[] -} - -/** - * @returns If an empty `docs` array is returned, it will be of type - * EmptyStream. In TypeScript, you should use `'empty' in docs` as - * a type guard for it. - */ -export function composeStream( - source: string, - forceDoc: boolean, - options?: Options -): Document.Parsed[] | EmptyStream { - const directives = new Directives({ - version: options?.version || defaultOptions.version || '1.2' - }) - const docs: Document.Parsed[] = [] - const lines: number[] = [] - - let atDirectives = false - let prelude: string[] = [] - let errors: YAMLParseError[] = [] - let warnings: YAMLWarning[] = [] - const onError = (offset: number, message: string, warning?: boolean) => { - warning - ? warnings.push(new YAMLWarning(offset, message)) - : errors.push(new YAMLParseError(offset, message)) - } - - const decorate = (doc: Document.Parsed, afterDoc: boolean) => { - const { comment, afterEmptyLine } = parsePrelude(prelude) - //console.log({ dc: doc.comment, prelude, comment }) - if (comment) { - const dc = doc.contents - if (afterDoc) { - doc.comment = doc.comment ? `${doc.comment}\n${comment}` : comment - } else if (afterEmptyLine || doc.directivesEndMarker || !dc) { - doc.commentBefore = comment - } else if ( - dc instanceof Collection && - (dc.type === 'MAP' || dc.type === 'SEQ') && - dc.items.length > 0 - ) { - const it = dc.items[0] as Node - const cb = it.commentBefore - it.commentBefore = cb ? `${comment}\n${cb}` : comment - } else { - const cb = dc.commentBefore - dc.commentBefore = cb ? `${comment}\n${cb}` : comment - } - } - - if (afterDoc) { - Array.prototype.push.apply(doc.errors, errors) - Array.prototype.push.apply(doc.warnings, warnings) - } else { - doc.errors = errors - doc.warnings = warnings - } - - prelude = [] - errors = [] - warnings = [] - } - - const parser = new CSTParser( - token => { - if (process.env.LOG_STREAM) console.dir(token, { depth: null }) - switch (token.type) { - case 'directive': - directives.add(token.source, onError) - prelude.push(token.source) - atDirectives = true - break - case 'document': { - const doc = composeDoc(options, directives, token, onError) - decorate(doc, false) - docs.push(doc) - atDirectives = false - break - } - case 'byte-order-mark': - case 'space': - break - case 'comment': - case 'newline': - prelude.push(token.source) - break - case 'error': { - const msg = token.source - ? `${token.message}: ${JSON.stringify(token.source)}` - : token.message - const error = new YAMLParseError(-1, msg) - if (atDirectives || docs.length === 0) errors.push(error) - else docs[docs.length - 1].errors.push(error) - break - } - case 'doc-end': { - const doc = docs[docs.length - 1] - if (!doc) { - const msg = 'Unexpected doc-end without preceding document' - errors.push(new YAMLParseError(token.offset, msg)) - break - } - const end = resolveEnd( - token.end, - token.offset + token.source.length, - doc.options.strict, - onError - ) - decorate(doc, true) - if (end.comment) { - const dc = doc.comment - doc.comment = dc ? `${dc}\n${end.comment}` : end.comment - } - doc.range[1] = end.offset - break - } - default: - errors.push(new YAMLParseError(-1, `Unsupported token ${token.type}`)) - } - }, - n => lines.push(n) - ) - parser.parse(source) - - if (docs.length === 0) { - if (forceDoc) { - const doc = new Document(undefined, options) as Document.Parsed - doc.directives = directives.atDocument() - if (atDirectives) - onError(source.length, 'Missing directives-end indicator line') - doc.setSchema() // FIXME: always do this in the constructor - doc.range = [0, source.length] - decorate(doc, false) - return [doc] - } else { - const { comment } = parsePrelude(prelude) - const empty: EmptyStream = Object.assign( - [], - { empty: true } as { empty: true }, - { comment, directives, errors, warnings } - ) - return empty - } - } - - decorate(docs[docs.length - 1], true) - return docs -} diff --git a/src/compose/composer.ts b/src/compose/composer.ts new file mode 100644 index 00000000..1e10e666 --- /dev/null +++ b/src/compose/composer.ts @@ -0,0 +1,191 @@ +import { Collection, Node } from '../ast/index.js' +import { Directives } from '../doc/directives.js' +import { Document } from '../doc/Document.js' +import { YAMLParseError, YAMLWarning } from '../errors.js' +import { defaultOptions, Options } from '../options.js' +import { Token } from '../parse/tokens.js' +import { composeDoc } from './compose-doc.js' +import { resolveEnd } from './resolve-end.js' + +function parsePrelude(prelude: string[]) { + let comment = '' + let atComment = false + let afterEmptyLine = false + for (let i = 0; i < prelude.length; ++i) { + const source = prelude[i] + switch (source[0]) { + case '#': + comment += + (comment === '' ? '' : afterEmptyLine ? '\n\n' : '\n') + + source.substring(1) + atComment = true + afterEmptyLine = false + break + case '%': + if (prelude[i + 1][0] !== '#') i += 1 + atComment = false + break + default: + // This may be wrong after doc-end, but in that case it doesn't matter + if (!atComment) afterEmptyLine = true + atComment = false + } + } + return { comment, afterEmptyLine } +} + +export class Composer { + lines: number[] = [] + + private directives: Directives + private doc: Document.Parsed | null = null + private onDocument: (doc: Document.Parsed) => void + private options: Options + private atDirectives = false + private prelude: string[] = [] + private errors: YAMLParseError[] = [] + private warnings: YAMLWarning[] = [] + + constructor(onDocument: Composer['onDocument'], options?: Options) { + this.directives = new Directives({ + version: options?.version || defaultOptions.version || '1.2' + }) + this.onDocument = onDocument + this.options = options || defaultOptions + } + + private onError = (offset: number, message: string, warning?: boolean) => { + if (warning) this.warnings.push(new YAMLWarning(offset, message)) + else this.errors.push(new YAMLParseError(offset, message)) + } + + private decorate(doc: Document.Parsed, afterDoc: boolean) { + const { comment, afterEmptyLine } = parsePrelude(this.prelude) + //console.log({ dc: doc.comment, prelude, comment }) + if (comment) { + const dc = doc.contents + if (afterDoc) { + doc.comment = doc.comment ? `${doc.comment}\n${comment}` : comment + } else if (afterEmptyLine || doc.directivesEndMarker || !dc) { + doc.commentBefore = comment + } else if ( + dc instanceof Collection && + (dc.type === 'MAP' || dc.type === 'SEQ') && + dc.items.length > 0 + ) { + const it = dc.items[0] as Node + const cb = it.commentBefore + it.commentBefore = cb ? `${comment}\n${cb}` : comment + } else { + const cb = dc.commentBefore + dc.commentBefore = cb ? `${comment}\n${cb}` : comment + } + } + + if (afterDoc) { + Array.prototype.push.apply(doc.errors, this.errors) + Array.prototype.push.apply(doc.warnings, this.warnings) + } else { + doc.errors = this.errors + doc.warnings = this.warnings + } + + this.prelude = [] + this.errors = [] + this.warnings = [] + } + + streamInfo() { + return { + comment: parsePrelude(this.prelude).comment, + directives: this.directives, + errors: this.errors, + warnings: this.warnings + } + } + + handleToken(token: Token) { + if (process.env.LOG_STREAM) console.dir(token, { depth: null }) + switch (token.type) { + case 'directive': + this.directives.add(token.source, this.onError) + this.prelude.push(token.source) + this.atDirectives = true + break + case 'document': { + const doc = composeDoc( + this.options, + this.directives, + token, + this.onError + ) + this.decorate(doc, false) + if (this.doc) this.onDocument(this.doc) + this.doc = doc + this.atDirectives = false + break + } + case 'byte-order-mark': + case 'space': + break + case 'comment': + case 'newline': + this.prelude.push(token.source) + break + case 'error': { + const msg = token.source + ? `${token.message}: ${JSON.stringify(token.source)}` + : token.message + const error = new YAMLParseError(-1, msg) + if (this.atDirectives || !this.doc) this.errors.push(error) + else this.doc.errors.push(error) + break + } + case 'doc-end': { + if (!this.doc) { + const msg = 'Unexpected doc-end without preceding document' + this.errors.push(new YAMLParseError(token.offset, msg)) + break + } + const end = resolveEnd( + token.end, + token.offset + token.source.length, + this.doc.options.strict, + this.onError + ) + this.decorate(this.doc, true) + if (end.comment) { + const dc = this.doc.comment + this.doc.comment = dc ? `${dc}\n${end.comment}` : end.comment + } + this.doc.range[1] = end.offset + break + } + default: + this.errors.push( + new YAMLParseError(-1, `Unsupported token ${token.type}`) + ) + } + } + + handleLine(n: number) { + this.lines.push(n) + } + + handleEnd(forceDoc = false, offset = -1) { + if (this.doc) { + this.decorate(this.doc, true) + this.onDocument(this.doc) + this.doc = null + } else if (forceDoc) { + const doc = new Document(undefined, this.options) as Document.Parsed + doc.directives = this.directives.atDocument() + if (this.atDirectives) + this.onError(offset, 'Missing directives-end indicator line') + doc.setSchema() // FIXME: always do this in the constructor + doc.range = [0, offset] + this.decorate(doc, false) + this.onDocument(doc) + } + } +} diff --git a/src/index.ts b/src/index.ts index 02b51cbe..cf8d090c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,30 +1,60 @@ -import { composeStream } from './compose/compose-stream.js' +import { Composer } from './compose/composer.js' import { LogLevel } from './constants.js' import { Document } from './doc/Document.js' import { YAMLParseError } from './errors.js' import { warn } from './log.js' import { Options } from './options.js' +import { CSTParser } from './parse/cst-parser.js' export { defaultOptions, scalarOptions } from './options.js' export { visit } from './visit.js' export { Document } -export function parseAllDocuments(src: string, options?: Options) { - return composeStream(src, false, options) +export interface EmptyStream + extends Array, + ReturnType { + empty: true } -export function parseDocument(src: string, options?: Options) { - const docs = composeStream(src, true, options) - if (docs.length === 0) return null - const doc = docs[0] - if ( - docs.length > 1 && - LogLevel.indexOf(doc.options.logLevel) >= LogLevel.ERROR - ) { - const errMsg = - 'Source contains multiple documents; please use YAML.parseAllDocuments()' - doc.errors.push(new YAMLParseError(docs[1].range[0], errMsg)) - } +/** + * @returns If an empty `docs` array is returned, it will be of type + * EmptyStream and contain additional stream information. In + * TypeScript, you should use `'empty' in docs` as a type guard for it. + */ +export function parseAllDocuments( + source: string, + options?: Options +): Document.Parsed[] | EmptyStream { + const docs: Document.Parsed[] = [] + const composer = new Composer(doc => docs.push(doc), options) + const parser = new CSTParser(token => composer.handleToken(token)) + parser.parse(source) + composer.handleEnd(false, source.length) + + if (docs.length > 0) return docs + return Object.assign< + Document.Parsed[], + { empty: true }, + ReturnType + >([], { empty: true }, composer.streamInfo()) +} + +export function parseDocument( + source: string, + options?: Options +): Document.Parsed | null { + let doc: Document.Parsed | null = null + const composer = new Composer(_doc => { + if (!doc) doc = _doc + else if (LogLevel.indexOf(doc.options.logLevel) >= LogLevel.ERROR) { + const errMsg = + 'Source contains multiple documents; please use YAML.parseAllDocuments()' + doc.errors.push(new YAMLParseError(_doc.range[0], errMsg)) + } + }, options) + const parser = new CSTParser(token => composer.handleToken(token)) + parser.parse(source) + composer.handleEnd(true, source.length) return doc } From bcc0420b7d16d9b34c4ad022e4f51e22dcdb1910 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Mon, 8 Feb 2021 18:02:51 +0200 Subject: [PATCH 83/89] Add lineCounter option to parse methods & LineCounter class --- src/compose/composer.ts | 6 ------ src/index.ts | 11 +++++++++-- src/options.d.ts | 9 +++++++++ src/options.js | 1 + src/parse/line-counter.ts | 36 ++++++++++++++++++++++++++++++++++++ tests/line-counter.js | 37 +++++++++++++++++++++++++++++++++++++ 6 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 src/parse/line-counter.ts create mode 100644 tests/line-counter.js diff --git a/src/compose/composer.ts b/src/compose/composer.ts index 1e10e666..c7b0c75a 100644 --- a/src/compose/composer.ts +++ b/src/compose/composer.ts @@ -35,8 +35,6 @@ function parsePrelude(prelude: string[]) { } export class Composer { - lines: number[] = [] - private directives: Directives private doc: Document.Parsed | null = null private onDocument: (doc: Document.Parsed) => void @@ -168,10 +166,6 @@ export class Composer { } } - handleLine(n: number) { - this.lines.push(n) - } - handleEnd(forceDoc = false, offset = -1) { if (this.doc) { this.decorate(this.doc, true) diff --git a/src/index.ts b/src/index.ts index cf8d090c..679a53aa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import { Options } from './options.js' import { CSTParser } from './parse/cst-parser.js' export { defaultOptions, scalarOptions } from './options.js' +export { LineCounter } from './parse/line-counter.js' export { visit } from './visit.js' export { Document } @@ -27,7 +28,10 @@ export function parseAllDocuments( ): Document.Parsed[] | EmptyStream { const docs: Document.Parsed[] = [] const composer = new Composer(doc => docs.push(doc), options) - const parser = new CSTParser(token => composer.handleToken(token)) + const parser = new CSTParser( + token => composer.handleToken(token), + options?.lineCounter?.addNewLine + ) parser.parse(source) composer.handleEnd(false, source.length) @@ -52,7 +56,10 @@ export function parseDocument( doc.errors.push(new YAMLParseError(_doc.range[0], errMsg)) } }, options) - const parser = new CSTParser(token => composer.handleToken(token)) + const parser = new CSTParser( + token => composer.handleToken(token), + options?.lineCounter?.addNewLine + ) parser.parse(source) composer.handleEnd(true, source.length) return doc diff --git a/src/options.d.ts b/src/options.d.ts index 07d22629..9180e8a1 100644 --- a/src/options.d.ts +++ b/src/options.d.ts @@ -1,6 +1,7 @@ import { Scalar } from './ast' import { LogLevelId } from './constants' import { Schema } from './doc/Schema' +import { LineCounter } from './parse/line-counter' /** * `yaml` defines document-specific options in three places: as an argument of @@ -49,6 +50,14 @@ export interface Options extends Schema.Options { * Default: `false` */ keepUndefined?: boolean + + /** + * If set, newlines will be tracked while parsing, to allow for + * `lineCounter.linePos(offset)` to provide the `{ line, col }` positions + * within the input. + */ + lineCounter?: LineCounter | null + /** * Control the logging level during parsing * diff --git a/src/options.js b/src/options.js index 16ac3ebb..3aab4eb9 100644 --- a/src/options.js +++ b/src/options.js @@ -15,6 +15,7 @@ export const defaultOptions = { keepCstNodes: false, keepNodeTypes: true, keepUndefined: false, + lineCounter: null, logLevel: 'warn', mapAsMap: false, maxAliasCount: 100, diff --git a/src/parse/line-counter.ts b/src/parse/line-counter.ts new file mode 100644 index 00000000..3d797ce2 --- /dev/null +++ b/src/parse/line-counter.ts @@ -0,0 +1,36 @@ +export class LineCounter { + lineStarts: number[] = [] + + /** + * Should be called in ascending order. Otherwise, call + * `lineCounter.lineStarts.sort()` before calling `linePos()`. + */ + addNewLine = (offset: number) => this.lineStarts.push(offset) + + // linePos = (offset: number) => { + // for (let i = this.lineStarts.length - 1; i >= 0; --i) { + // const start = this.lineStarts[i] + // if (start <= offset) return { line: i + 1, col: offset - start + 1 } + // } + // return { line: 0, col: offset } + // } + + /** + * Performs a binary search and returns the 1-indexed { line, col } + * position of `offset`. If `line === 0`, `addNewLine` has never been + * called or `offset` is before the first known newline. + */ + linePos = (offset: number) => { + let low = 0 + let high = this.lineStarts.length + while (low < high) { + const mid = (low + high) >> 1 // Math.floor((low + high) / 2) + if (this.lineStarts[mid] < offset) low = mid + 1 + else high = mid + } + if (low === 0) return { line: 0, col: offset } + if (this.lineStarts[low] === offset) return { line: low + 1, col: 1 } + const start = this.lineStarts[low - 1] + return { line: low, col: offset - start + 1 } + } +} diff --git a/tests/line-counter.js b/tests/line-counter.js new file mode 100644 index 00000000..366e9004 --- /dev/null +++ b/tests/line-counter.js @@ -0,0 +1,37 @@ +import * as YAML from '../index.js' +const { LineCounter, parseDocument } = YAML + +test('Parse error, no newlines', () => { + const lineCounter = new LineCounter() + const doc = parseDocument('foo: bar: baz', { lineCounter }) + expect(doc.errors).toMatchObject([{ offset: 5 }]) + expect(lineCounter.lineStarts).toMatchObject([0]) + expect(lineCounter.linePos(5)).toMatchObject({ line: 1, col: 6 }) +}) + +test('Parse error with newlines', () => { + const lineCounter = new LineCounter() + const doc = parseDocument('foo:\n bar: - baz\n', { lineCounter }) + expect(doc.errors).toMatchObject([{ offset: 14 }]) + expect(lineCounter.lineStarts).toMatchObject([0, 5, 18]) + expect(lineCounter.linePos(14)).toMatchObject({ line: 2, col: 10 }) +}) + +test('block scalar', () => { + const lineCounter = new LineCounter() + const doc = parseDocument('foo: |\n a\n b\n c\nbar:\n baz\n', { lineCounter }) + expect(lineCounter.lineStarts).toMatchObject([0, 7, 10, 13, 16, 21, 26]) + for (const { offset, line, col } of [ + { offset: 10, line: 3, col: 1 }, + { offset: 11, line: 3, col: 2 }, + { offset: 12, line: 3, col: 3 }, + { offset: 13, line: 4, col: 1 }, + { offset: 14, line: 4, col: 2 }, + { offset: 15, line: 4, col: 3 }, + { offset: 16, line: 5, col: 1 }, + { offset: 17, line: 5, col: 2 }, + { offset: 18, line: 5, col: 3 } + ]) { + expect(lineCounter.linePos(offset)).toMatchObject({ line, col }) + } +}) From a6d9bcffcafffe6f7dfe42e6c75512060e359a2d Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Fri, 12 Feb 2021 09:46:52 +0200 Subject: [PATCH 84/89] Re-rename CSTParser as just Parser --- src/compose/compose-collection.ts | 6 +++++- src/index.ts | 6 +++--- src/parse/cst-stream.ts | 6 +++--- src/parse/{cst-parser.ts => parser.ts} | 2 +- src/parse/test.ts | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) rename src/parse/{cst-parser.ts => parser.ts} (99%) diff --git a/src/compose/compose-collection.ts b/src/compose/compose-collection.ts index 86a0031e..89677b11 100644 --- a/src/compose/compose-collection.ts +++ b/src/compose/compose-collection.ts @@ -1,6 +1,10 @@ import { Node, Scalar, YAMLMap, YAMLSeq } from '../ast/index.js' import type { Document } from '../doc/Document.js' -import type { BlockMap, BlockSequence, FlowCollection } from '../parse/tokens.js' +import type { + BlockMap, + BlockSequence, + FlowCollection +} from '../parse/tokens.js' import { resolveBlockMap } from './resolve-block-map.js' import { resolveBlockSeq } from './resolve-block-seq.js' import { resolveFlowCollection } from './resolve-flow-collection.js' diff --git a/src/index.ts b/src/index.ts index 679a53aa..f22d4637 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,7 +4,7 @@ import { Document } from './doc/Document.js' import { YAMLParseError } from './errors.js' import { warn } from './log.js' import { Options } from './options.js' -import { CSTParser } from './parse/cst-parser.js' +import { Parser } from './parse/parser.js' export { defaultOptions, scalarOptions } from './options.js' export { LineCounter } from './parse/line-counter.js' @@ -28,7 +28,7 @@ export function parseAllDocuments( ): Document.Parsed[] | EmptyStream { const docs: Document.Parsed[] = [] const composer = new Composer(doc => docs.push(doc), options) - const parser = new CSTParser( + const parser = new Parser( token => composer.handleToken(token), options?.lineCounter?.addNewLine ) @@ -56,7 +56,7 @@ export function parseDocument( doc.errors.push(new YAMLParseError(_doc.range[0], errMsg)) } }, options) - const parser = new CSTParser( + const parser = new Parser( token => composer.handleToken(token), options?.lineCounter?.addNewLine ) diff --git a/src/parse/cst-stream.ts b/src/parse/cst-stream.ts index 1a23e43b..60542bcd 100644 --- a/src/parse/cst-stream.ts +++ b/src/parse/cst-stream.ts @@ -1,6 +1,6 @@ import { Transform, TransformOptions } from 'stream' import { StringDecoder } from 'string_decoder' -import { CSTParser } from './cst-parser.js' +import { Parser } from './parser.js' export type ParseStreamOptions = Omit< TransformOptions, @@ -9,7 +9,7 @@ export type ParseStreamOptions = Omit< export class CSTStream extends Transform { decoder: StringDecoder - parser: CSTParser + parser: Parser constructor(options: ParseStreamOptions = {}) { super({ @@ -19,7 +19,7 @@ export class CSTStream extends Transform { objectMode: true }) this.decoder = new StringDecoder(options.defaultEncoding || 'utf8') - this.parser = new CSTParser(token => this.push(token)) + this.parser = new Parser(token => this.push(token)) } _flush(done: (error?: Error) => void) { diff --git a/src/parse/cst-parser.ts b/src/parse/parser.ts similarity index 99% rename from src/parse/cst-parser.ts rename to src/parse/parser.ts index 72169155..79f3d0c4 100644 --- a/src/parse/cst-parser.ts +++ b/src/parse/parser.ts @@ -113,7 +113,7 @@ function getFirstKeyStartProps(prev: SourceToken[]) { } /** A YAML concrete syntax tree parser */ -export class CSTParser { +export class Parser { private push: (token: Token) => void private onNewLine?: (offset: number) => void diff --git a/src/parse/test.ts b/src/parse/test.ts index 94f899e4..3a435a70 100644 --- a/src/parse/test.ts +++ b/src/parse/test.ts @@ -1,5 +1,5 @@ import { CSTStream } from './cst-stream.js' -import { CSTParser } from './cst-parser.js' +import { Parser } from './parser.js' export function stream(source: string) { const ps = new CSTStream().on('data', d => console.dir(d, { depth: null })) @@ -9,7 +9,7 @@ export function stream(source: string) { export function test(source: string) { const lines: number[] = [] - const parser = new CSTParser( + const parser = new Parser( t => console.dir(t, { depth: null }), n => lines.push(n) ) From 9dd0cb1920b3716fddf6d21180da28cc9196989d Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Fri, 12 Feb 2021 16:32:50 +0200 Subject: [PATCH 85/89] Add exports & API docs for Lexer, Parser & Composer --- src/compose/composer.ts | 41 +++++++++++++++++++++++++++++++--- src/index.ts | 18 ++++++--------- src/parse/lexer.ts | 15 +++++++++++++ src/parse/line-counter.ts | 13 +++++------ src/parse/parser.ts | 46 ++++++++++++++++++++++++++++++--------- src/parse/tokens.ts | 2 ++ 6 files changed, 103 insertions(+), 32 deletions(-) diff --git a/src/compose/composer.ts b/src/compose/composer.ts index c7b0c75a..691c510d 100644 --- a/src/compose/composer.ts +++ b/src/compose/composer.ts @@ -3,7 +3,7 @@ import { Directives } from '../doc/directives.js' import { Document } from '../doc/Document.js' import { YAMLParseError, YAMLWarning } from '../errors.js' import { defaultOptions, Options } from '../options.js' -import { Token } from '../parse/tokens.js' +import type { Token } from '../parse/tokens.js' import { composeDoc } from './compose-doc.js' import { resolveEnd } from './resolve-end.js' @@ -34,6 +34,18 @@ function parsePrelude(prelude: string[]) { return { comment, afterEmptyLine } } +/** + * Compose a stream of CST nodes into a stream of YAML Documents. + * + * ```ts + * const options: Options = { ... } + * const docs: Document.Parsed[] = [] + * const composer = new Composer(doc => docs.push(doc), options) + * const parser = new Parser(composer.next) + * parser.parse(source) + * composer.end() + * ``` + */ export class Composer { private directives: Directives private doc: Document.Parsed | null = null @@ -93,6 +105,11 @@ export class Composer { this.warnings = [] } + /** + * Current stream status information. + * + * Mostly useful at the end of input for an empty stream. + */ streamInfo() { return { comment: parsePrelude(this.prelude).comment, @@ -102,7 +119,11 @@ export class Composer { } } - handleToken(token: Token) { + /** + * Advance the composed by one CST token. Bound to the Composer + * instance, so may be used directly as a callback function. + */ + next = (token: Token) => { if (process.env.LOG_STREAM) console.dir(token, { depth: null }) switch (token.type) { case 'directive': @@ -166,7 +187,21 @@ export class Composer { } } - handleEnd(forceDoc = false, offset = -1) { + /** Call at end of input to push out any remaining document. */ + end(): void + + /** + * Call at end of input to push out any remaining document. + * + * @param forceDoc - If the stream contains no document, still emit a final + * document including any comments and directives that would be applied + * to a subsequent document. + * @param offset - Should be set if `forceDoc` is also set, to set the + * document range end and to indicate errors correctly. + */ + end(forceDoc: true, offset: number): void + + end(forceDoc = false, offset = -1) { if (this.doc) { this.decorate(this.doc, true) this.onDocument(this.doc) diff --git a/src/index.ts b/src/index.ts index f22d4637..fe74d0e1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,9 +7,11 @@ import { Options } from './options.js' import { Parser } from './parse/parser.js' export { defaultOptions, scalarOptions } from './options.js' +export { Lexer } from './parse/lexer.js' export { LineCounter } from './parse/line-counter.js' +export * as tokens from './parse/tokens.js' export { visit } from './visit.js' -export { Document } +export { Composer, Document, Parser } export interface EmptyStream extends Array, @@ -28,12 +30,9 @@ export function parseAllDocuments( ): Document.Parsed[] | EmptyStream { const docs: Document.Parsed[] = [] const composer = new Composer(doc => docs.push(doc), options) - const parser = new Parser( - token => composer.handleToken(token), - options?.lineCounter?.addNewLine - ) + const parser = new Parser(composer.next, options?.lineCounter?.addNewLine) parser.parse(source) - composer.handleEnd(false, source.length) + composer.end() if (docs.length > 0) return docs return Object.assign< @@ -56,12 +55,9 @@ export function parseDocument( doc.errors.push(new YAMLParseError(_doc.range[0], errMsg)) } }, options) - const parser = new Parser( - token => composer.handleToken(token), - options?.lineCounter?.addNewLine - ) + const parser = new Parser(composer.next, options?.lineCounter?.addNewLine) parser.parse(source) - composer.handleEnd(true, source.length) + composer.end(true, source.length) return doc } diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts index f9bda91b..fe5628a3 100644 --- a/src/parse/lexer.ts +++ b/src/parse/lexer.ts @@ -96,6 +96,21 @@ const invalidIdentifierChars = [' ', ',', '[', ']', '{', '}', '\n', '\r', '\t'] const isNotIdentifierChar = (ch: string) => !ch || invalidIdentifierChars.includes(ch) +/** + * Splits an input string into lexical tokens, i.e. smaller strings that are + * easily identifiable by `tokens.tokenType()`. + * + * Lexing starts always in a "stream" context. Incomplete input may be buffered + * until a complete token can be emitted. + * + * In addition to slices of the original input, the following control characters + * may also be emitted: + * + * - `\x02` (Start of Text): A document starts with the next token + * - `\x18` (Cancel): Unexpected end of flow-mode (indicates an error) + * - `\x1f` (Unit Separator): Next token is a scalar value + * - `\u{FEFF}` (Byte order mark): Emitted separately outside documents + */ export class Lexer { private push: (token: string) => void diff --git a/src/parse/line-counter.ts b/src/parse/line-counter.ts index 3d797ce2..1b308f31 100644 --- a/src/parse/line-counter.ts +++ b/src/parse/line-counter.ts @@ -1,3 +1,8 @@ +/** + * Tracks newlines during parsing in order to provide an efficient API for + * determining the one-indexed `{ line, col }` position for any offset + * within the input. + */ export class LineCounter { lineStarts: number[] = [] @@ -7,14 +12,6 @@ export class LineCounter { */ addNewLine = (offset: number) => this.lineStarts.push(offset) - // linePos = (offset: number) => { - // for (let i = this.lineStarts.length - 1; i >= 0; --i) { - // const start = this.lineStarts[i] - // if (start <= offset) return { line: i + 1, col: offset - start + 1 } - // } - // return { line: 0, col: offset } - // } - /** * Performs a binary search and returns the 1-indexed { line, col } * position of `offset`. If `line === 0`, `addNewLine` has never been diff --git a/src/parse/parser.ts b/src/parse/parser.ts index 79f3d0c4..c6993b6a 100644 --- a/src/parse/parser.ts +++ b/src/parse/parser.ts @@ -112,13 +112,27 @@ function getFirstKeyStartProps(prev: SourceToken[]) { return prev.splice(i, prev.length) } -/** A YAML concrete syntax tree parser */ +/** + * A YAML concrete syntax tree (CST) parser + * + * While the `parse()` method provides an API for parsing a source string + * directly, the parser may also be used with a user-provided lexer: + * + * ```ts + * const cst: Token[] = [] + * const parser = new Parser(tok => cst.push(tok)) + * const src: string = ... + * + * // The following would be equivalent to `parser.parse(src, false)` + * const lexer = new Lexer(parser.next) + * lexer.lex(src, false) + * parser.end() + * ``` + */ export class Parser { private push: (token: Token) => void private onNewLine?: (offset: number) => void - private lexer = new Lexer(ts => this.token(ts)) - /** If true, space and sequence indicators count as indentation */ private atNewLine = true @@ -145,7 +159,8 @@ export class Parser { /** * @param push - Called separately with each parsed token - * @param onNewLine - If defined, called separately with the start position of each new line + * @param onNewLine - If defined, called separately with the start position of + * each new line (in `parse()`, including the start of input). * @public */ constructor( @@ -157,9 +172,9 @@ export class Parser { } /** - * Parse `source` as a YAML stream, calling `push` with each - * directive, document and other structure as it is completely parsed. - * If `incomplete`, a part of the last line may be left as a buffer for the next call. + * Parse `source` as a YAML stream, calling `push` with each directive, + * document and other structure as it is completely parsed. If `incomplete`, + * a part of the last line may be left as a buffer for the next call. * * Errors are not thrown, but pushed out as `{ type: 'error', message }` tokens. * @public @@ -167,11 +182,14 @@ export class Parser { parse(source: string, incomplete = false) { if (this.onNewLine && this.offset === 0) this.onNewLine(0) this.lexer.lex(source, incomplete) - if (!incomplete) while (this.stack.length > 0) this.pop() + if (!incomplete) this.end() } - /** Advance the parser by the `source` of one lexical token. */ - token(source: string) { + /** + * Advance the parser by the `source` of one lexical token. Bound to the + * Parser instance, so may be used directly as a callback function. + */ + next = (source: string) => { this.source = source if (process.env.LOG_TOKENS) console.log('|', prettyToken(source)) @@ -217,6 +235,14 @@ export class Parser { } } + // Must be defined after `next()` + private lexer = new Lexer(this.next) + + /** Call at end of input to push out any remaining constructions */ + end() { + while (this.stack.length > 0) this.pop() + } + private get sourceToken() { return { type: this.type, diff --git a/src/parse/tokens.ts b/src/parse/tokens.ts index fe088cd1..2dd09bce 100644 --- a/src/parse/tokens.ts +++ b/src/parse/tokens.ts @@ -128,6 +128,7 @@ export const FLOW_END = '\x18' // C0: Cancel /** Next token is a scalar value */ export const SCALAR = '\x1f' // C0: Unit Separator +/** Get a printable representation of a lexer token */ export function prettyToken(token: string) { switch (token) { case BOM: @@ -143,6 +144,7 @@ export function prettyToken(token: string) { } } +/** Identify the type of a lexer token. May return `null` for unknown tokens. */ export function tokenType(source: string): SourceTokenType | null { switch (source) { case BOM: From cef84899d339f915ca53324cb4b442be373cbdf0 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 13 Feb 2021 11:41:04 +0200 Subject: [PATCH 86/89] Refactor options & tags/options as TypeScript --- rollup.dev-config.js | 9 +- src/doc/Schema.d.ts | 2 +- src/options.js | 87 ------------- src/{options.d.ts => options.ts} | 205 ++++++++++++++----------------- src/tags/options.js | 26 ---- src/tags/options.ts | 109 ++++++++++++++++ 6 files changed, 208 insertions(+), 230 deletions(-) delete mode 100644 src/options.js rename src/{options.d.ts => options.ts} (51%) delete mode 100644 src/tags/options.js create mode 100644 src/tags/options.ts diff --git a/rollup.dev-config.js b/rollup.dev-config.js index cb3b1f20..0589b377 100644 --- a/rollup.dev-config.js +++ b/rollup.dev-config.js @@ -2,8 +2,13 @@ import { resolve } from 'path' import babel from '@rollup/plugin-babel' export default { - input: ['src/ast/index.js', 'src/doc/Document.js', 'src/options.js'], - external: [resolve('src/doc/directives.js'), resolve('src/errors.js')], + input: ['src/ast/index.js', 'src/doc/Document.js'], + external: [ + resolve('src/doc/directives.js'), + resolve('src/errors.js'), + resolve('src/options.js'), + resolve('src/tags/options.js') + ], output: { dir: 'lib', format: 'cjs', esModule: false, preserveModules: true }, plugins: [babel()] } diff --git a/src/doc/Schema.d.ts b/src/doc/Schema.d.ts index 19503a41..c60ae6e8 100644 --- a/src/doc/Schema.d.ts +++ b/src/doc/Schema.d.ts @@ -17,7 +17,7 @@ export namespace Schema { * Array of additional tags to include in the schema, or a function that may * modify the schema's base tag array. */ - customTags?: (TagId | Tag)[] | ((tags: Tag[]) => Tag[]) + customTags?: (TagId | Tag)[] | ((tags: Tag[]) => Tag[]) | null /** * Enable support for `<<` merge keys. * diff --git a/src/options.js b/src/options.js deleted file mode 100644 index 3aab4eb9..00000000 --- a/src/options.js +++ /dev/null @@ -1,87 +0,0 @@ -import { defaultTagPrefix } from './constants.js' -import { - binaryOptions, - boolOptions, - intOptions, - nullOptions, - strOptions -} from './tags/options.js' - -export const defaultOptions = { - anchorPrefix: 'a', - customTags: null, - indent: 2, - indentSeq: true, - keepCstNodes: false, - keepNodeTypes: true, - keepUndefined: false, - lineCounter: null, - logLevel: 'warn', - mapAsMap: false, - maxAliasCount: 100, - prettyErrors: true, - simpleKeys: false, - strict: true, - version: '1.2' -} - -export const scalarOptions = { - get binary() { - return binaryOptions - }, - set binary(opt) { - Object.assign(binaryOptions, opt) - }, - get bool() { - return boolOptions - }, - set bool(opt) { - Object.assign(boolOptions, opt) - }, - get int() { - return intOptions - }, - set int(opt) { - Object.assign(intOptions, opt) - }, - get null() { - return nullOptions - }, - set null(opt) { - Object.assign(nullOptions, opt) - }, - get str() { - return strOptions - }, - set str(opt) { - Object.assign(strOptions, opt) - } -} - -export const documentOptions = { - '1.0': { - schema: 'yaml-1.1', - merge: true, - tagPrefixes: [ - { handle: '!', prefix: defaultTagPrefix }, - { handle: '!!', prefix: 'tag:private.yaml.org,2002:' } - ] - }, - 1.1: { - schema: 'yaml-1.1', - merge: true, - tagPrefixes: [ - { handle: '!', prefix: '!' }, - { handle: '!!', prefix: defaultTagPrefix } - ] - }, - 1.2: { - schema: 'core', - merge: false, - resolveKnownTags: true, - tagPrefixes: [ - { handle: '!', prefix: '!' }, - { handle: '!!', prefix: defaultTagPrefix } - ] - } -} diff --git a/src/options.d.ts b/src/options.ts similarity index 51% rename from src/options.d.ts rename to src/options.ts index 9180e8a1..6451eb75 100644 --- a/src/options.d.ts +++ b/src/options.ts @@ -1,16 +1,13 @@ -import { Scalar } from './ast' -import { LogLevelId } from './constants' -import { Schema } from './doc/Schema' -import { LineCounter } from './parse/line-counter' - -/** - * `yaml` defines document-specific options in three places: as an argument of - * parse, create and stringify calls, in the values of `YAML.defaultOptions`, - * and in the version-dependent `YAML.Document.defaults` object. Values set in - * `YAML.defaultOptions` override version-dependent defaults, and argument - * options override both. - */ -export const defaultOptions: Options +import { LogLevelId, defaultTagPrefix } from './constants.js' +import type { Schema } from './doc/Schema.js' +import type { LineCounter } from './parse/line-counter.js' +import { + binaryOptions, + boolOptions, + intOptions, + nullOptions, + strOptions +} from './tags/options.js' export interface Options extends Schema.Options { /** @@ -104,111 +101,91 @@ export interface Options extends Schema.Options { } /** - * Some customization options are availabe to control the parsing and - * stringification of scalars. Note that these values are used by all documents. + * `yaml` defines document-specific options in three places: as an argument of + * parse, create and stringify calls, in the values of `YAML.defaultOptions`, + * and in the version-dependent `YAML.Document.defaults` object. Values set in + * `YAML.defaultOptions` override version-dependent defaults, and argument + * options override both. */ -export const scalarOptions: { - binary: scalarOptions.Binary - bool: scalarOptions.Bool - int: scalarOptions.Int - null: scalarOptions.Null - str: scalarOptions.Str +export const defaultOptions: Options = { + anchorPrefix: 'a', + customTags: null, + indent: 2, + indentSeq: true, + keepCstNodes: false, + keepNodeTypes: true, + keepUndefined: false, + lineCounter: null, + logLevel: 'warn', + mapAsMap: false, + maxAliasCount: 100, + prettyErrors: true, + simpleKeys: false, + strict: true, + version: '1.2' } -export namespace scalarOptions { - interface Binary { - /** - * The type of string literal used to stringify `!!binary` values. - * - * Default: `'BLOCK_LITERAL'` - */ - defaultType: Scalar.Type - /** - * Maximum line width for `!!binary`. - * - * Default: `76` - */ - lineWidth: number - } - - interface Bool { - /** - * String representation for `true`. With the core schema, use `'true' | 'True' | 'TRUE'`. - * - * Default: `'true'` - */ - trueStr: string - /** - * String representation for `false`. With the core schema, use `'false' | 'False' | 'FALSE'`. - * - * Default: `'false'` - */ - falseStr: string - } - - interface Int { - /** - * Whether integers should be parsed into BigInt values. - * - * Default: `false` - */ - asBigInt: boolean - } - interface Null { - /** - * String representation for `null`. With the core schema, use `'null' | 'Null' | 'NULL' | '~' | ''`. - * - * Default: `'null'` - */ - nullStr: string +/** + * Some customization options are availabe to control the parsing and + * stringification of scalars. Note that these values are used by all documents. + */ +export const scalarOptions = { + get binary() { + return binaryOptions + }, + set binary(opt) { + Object.assign(binaryOptions, opt) + }, + get bool() { + return boolOptions + }, + set bool(opt) { + Object.assign(boolOptions, opt) + }, + get int() { + return intOptions + }, + set int(opt) { + Object.assign(intOptions, opt) + }, + get null() { + return nullOptions + }, + set null(opt) { + Object.assign(nullOptions, opt) + }, + get str() { + return strOptions + }, + set str(opt) { + Object.assign(strOptions, opt) } +} - interface Str { - /** - * The default type of string literal used to stringify values in general - * - * Default: `'PLAIN'` - */ - defaultType: Scalar.Type - /** - * The default type of string literal used to stringify implicit key values - * - * Default: `'PLAIN'` - */ - defaultKeyType: Scalar.Type - /** - * Use 'single quote' rather than "double quote" by default - * - * Default: `false` - */ - defaultQuoteSingle: boolean - doubleQuoted: { - /** - * Whether to restrict double-quoted strings to use JSON-compatible syntax. - * - * Default: `false` - */ - jsonEncoding: boolean - /** - * Minimum length to use multiple lines to represent the value. - * - * Default: `40` - */ - minMultiLineLength: number - } - fold: { - /** - * Maximum line width (set to `0` to disable folding). - * - * Default: `80` - */ - lineWidth: number - /** - * Minimum width for highly-indented content. - * - * Default: `20` - */ - minContentWidth: number - } +export const documentOptions = { + '1.0': { + schema: 'yaml-1.1', + merge: true, + tagPrefixes: [ + { handle: '!', prefix: defaultTagPrefix }, + { handle: '!!', prefix: 'tag:private.yaml.org,2002:' } + ] + }, + 1.1: { + schema: 'yaml-1.1', + merge: true, + tagPrefixes: [ + { handle: '!', prefix: '!' }, + { handle: '!!', prefix: defaultTagPrefix } + ] + }, + 1.2: { + schema: 'core', + merge: false, + resolveKnownTags: true, + tagPrefixes: [ + { handle: '!', prefix: '!' }, + { handle: '!!', prefix: defaultTagPrefix } + ] } } diff --git a/src/tags/options.js b/src/tags/options.js deleted file mode 100644 index 00e44390..00000000 --- a/src/tags/options.js +++ /dev/null @@ -1,26 +0,0 @@ -import { Type } from '../constants.js' - -export const binaryOptions = { - defaultType: Type.BLOCK_LITERAL, - lineWidth: 76 -} - -export const boolOptions = { trueStr: 'true', falseStr: 'false' } - -export const intOptions = { asBigInt: false } - -export const nullOptions = { nullStr: 'null' } - -export const strOptions = { - defaultType: Type.PLAIN, - defaultKeyType: Type.PLAIN, - defaultQuoteSingle: false, - doubleQuoted: { - jsonEncoding: false, - minMultiLineLength: 40 - }, - fold: { - lineWidth: 80, - minContentWidth: 20 - } -} diff --git a/src/tags/options.ts b/src/tags/options.ts new file mode 100644 index 00000000..80ab4dab --- /dev/null +++ b/src/tags/options.ts @@ -0,0 +1,109 @@ +import { Type } from '../constants.js' + +export const binaryOptions = { + /** + * The type of string literal used to stringify `!!binary` values. + * + * Default: `'BLOCK_LITERAL'` + */ + defaultType: Type.BLOCK_LITERAL, + + /** + * Maximum line width for `!!binary`. + * + * Default: `76` + */ + lineWidth: 76 +} + +export const boolOptions = { + /** + * String representation for `true`. + * With the core schema, use `true`, `True`, or `TRUE`. + * + * Default: `'true'` + */ + trueStr: 'true', + + /** + * String representation for `false`. + * With the core schema, use `false`, `False`, or `FALSE`. + * + * Default: `'false'` + */ + falseStr: 'false' +} + +export const intOptions = { + /** + * Whether integers should be parsed into BigInt values. + * + * Default: `false` + */ + asBigInt: false +} + +export const nullOptions = { + /** + * String representation for `null`. + * With the core schema, use `null`, `Null`, `NULL`, `~`, or an empty string. + * + * Default: `'null'` + */ + nullStr: 'null' +} + +export const strOptions = { + /** + * The default type of string literal used to stringify values in general + * + * Default: `'PLAIN'` + */ + defaultType: Type.PLAIN, + + /** + * The default type of string literal used to stringify implicit key values + * + * Default: `'PLAIN'` + */ + defaultKeyType: Type.PLAIN, + + /** + * Use 'single quote' rather than "double quote" by default + * + * Default: `false` + */ + defaultQuoteSingle: false, + + doubleQuoted: { + /** + * Whether to restrict double-quoted strings to use JSON-compatible syntax. + * + * Default: `false` + */ + jsonEncoding: false, + + /** + * Minimum length to use multiple lines to represent the value. + * + * Default: `40` + */ + minMultiLineLength: 40 + }, + + fold: { + /** + * Maximum line width (set to `0` to disable folding). + * + * Default: `80` + */ + lineWidth: 80, + + /** + * Minimum width for highly-indented content. + * + * Default: `20` + */ + minContentWidth: 20 + } +} From ff8bdcaae31737357dfabefaffc542619228bac7 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 13 Feb 2021 13:40:54 +0200 Subject: [PATCH 87/89] Include TS support in Node.js & browser builds; drop separate dev build --- package-lock.json | 41 ++++++++++++++++++++++++++++++++++++++++ package.json | 7 ++++--- rollup.browser-config.js | 11 +++++++++-- rollup.dev-config.js | 14 -------------- rollup.node-config.js | 6 ++++-- tsconfig.json | 2 -- 6 files changed, 58 insertions(+), 23 deletions(-) delete mode 100644 rollup.dev-config.js diff --git a/package-lock.json b/package-lock.json index e0afad3b..6e2fe5f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2182,6 +2182,26 @@ "@rollup/pluginutils": "^3.1.0" } }, + "@rollup/plugin-replace": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/@rollup/plugin-replace/-/plugin-replace-2.3.4.tgz", + "integrity": "sha512-waBhMzyAtjCL1GwZes2jaE9MjuQ/DQF2BatH3fRivUF3z0JBFrU0U6iBNC/4WR+2rLKhaAhPWDNPYp4mI6RqdQ==", + "dev": true, + "requires": { + "@rollup/pluginutils": "^3.1.0", + "magic-string": "^0.25.7" + } + }, + "@rollup/plugin-typescript": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/@rollup/plugin-typescript/-/plugin-typescript-8.1.1.tgz", + "integrity": "sha512-DPFy0SV8/GgHFL31yPFVo0G1T3yzwdw6R9KisBfO2zCYbDHUqDChSWr1KmtpGz/TmutpoGJjIvu80p9HzCEF0A==", + "dev": true, + "requires": { + "@rollup/pluginutils": "^3.1.0", + "resolve": "^1.17.0" + } + }, "@rollup/pluginutils": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-3.1.0.tgz", @@ -6028,6 +6048,15 @@ "yallist": "^4.0.0" } }, + "magic-string": { + "version": "0.25.7", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.7.tgz", + "integrity": "sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA==", + "dev": true, + "requires": { + "sourcemap-codec": "^1.4.4" + } + }, "make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -7352,6 +7381,12 @@ "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=", "dev": true }, + "sourcemap-codec": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", + "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==", + "dev": true + }, "spdx-correct": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", @@ -7700,6 +7735,12 @@ "punycode": "^2.1.1" } }, + "tslib": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz", + "integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==", + "dev": true + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", diff --git a/package.json b/package.json index c54f2f6a..535d90d2 100644 --- a/package.json +++ b/package.json @@ -53,13 +53,11 @@ "scripts": { "build": "npm run build:node && npm run build:browser", "build:browser": "rollup -c rollup.browser-config.js", - "prebuild:dev": "rollup -c rollup.dev-config.js", - "build:dev": "tsc", "build:node": "rollup -c rollup.node-config.js", "clean": "git clean -fdxe node_modules", "lint": "eslint src/", "prettier": "prettier --write .", - "prestart": "tsc", + "prestart": "npm run build:node", "start": "node -i -e 'YAML=require(\"./lib/index.js\")'", "test": "jest", "test:browsers": "cd playground && npm test", @@ -85,6 +83,8 @@ "@babel/preset-env": "^7.12.11", "@babel/preset-typescript": "^7.12.7", "@rollup/plugin-babel": "^5.2.3", + "@rollup/plugin-replace": "^2.3.4", + "@rollup/plugin-typescript": "^8.1.1", "babel-eslint": "^10.1.0", "babel-jest": "^26.6.3", "babel-plugin-trace": "^1.1.0", @@ -97,6 +97,7 @@ "jest-ts-webcompat-resolver": "^1.0.0", "prettier": "^2.2.1", "rollup": "^2.38.2", + "tslib": "^2.1.0", "typescript": "^4.1.3" }, "engines": { diff --git a/rollup.browser-config.js b/rollup.browser-config.js index 75ac506a..211e2186 100644 --- a/rollup.browser-config.js +++ b/rollup.browser-config.js @@ -1,17 +1,24 @@ import babel from '@rollup/plugin-babel' +import replace from '@rollup/plugin-replace' +import typescript from '@rollup/plugin-typescript' export default { input: { - index: 'src/index.js', + index: 'src/index.ts', types: 'src/types.js', util: 'src/util.js' }, output: { dir: 'browser/dist', format: 'esm', preserveModules: true }, plugins: [ + replace({ + 'process.env.LOG_TOKENS': String(!!process.env.LOG_TOKENS), + 'process.env.LOG_STREAM': String(!!process.env.LOG_STREAM) + }), babel({ babelHelpers: 'bundled', presets: [['@babel/env', { modules: false }]] - }) + }), + typescript() ], treeshake: { moduleSideEffects: false, propertyReadSideEffects: false } } diff --git a/rollup.dev-config.js b/rollup.dev-config.js deleted file mode 100644 index 0589b377..00000000 --- a/rollup.dev-config.js +++ /dev/null @@ -1,14 +0,0 @@ -import { resolve } from 'path' -import babel from '@rollup/plugin-babel' - -export default { - input: ['src/ast/index.js', 'src/doc/Document.js'], - external: [ - resolve('src/doc/directives.js'), - resolve('src/errors.js'), - resolve('src/options.js'), - resolve('src/tags/options.js') - ], - output: { dir: 'lib', format: 'cjs', esModule: false, preserveModules: true }, - plugins: [babel()] -} diff --git a/rollup.node-config.js b/rollup.node-config.js index f6f3f561..8b194dae 100644 --- a/rollup.node-config.js +++ b/rollup.node-config.js @@ -1,8 +1,9 @@ import babel from '@rollup/plugin-babel' +import typescript from '@rollup/plugin-typescript' export default { input: { - index: 'src/index.js', + index: 'src/index.ts', 'test-events': 'src/test-events.js', types: 'src/types.js', util: 'src/util.js' @@ -17,7 +18,8 @@ export default { babel({ babelHelpers: 'bundled', presets: [['@babel/env', { modules: false, targets: { node: '10.0' } }]] - }) + }), + typescript() ], treeshake: { moduleSideEffects: false, propertyReadSideEffects: false } } diff --git a/tsconfig.json b/tsconfig.json index 67eca3be..5a45dd9c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,10 +1,8 @@ { "compilerOptions": { - "module": "CommonJS", "moduleResolution": "node", "noUnusedLocals": true, "noUnusedParameters": true, - "outDir": "lib/", "strict": true, "target": "ES2017" }, From 3ac4a332ec6b90315e8c81c43fc367f379ea9a74 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sat, 13 Feb 2021 13:52:59 +0200 Subject: [PATCH 88/89] ci: Re-enable browser & dist tests --- .github/workflows/browsers.yml | 2 +- .github/workflows/nodejs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/browsers.yml b/.github/workflows/browsers.yml index 8b8cd7a9..72f87908 100644 --- a/.github/workflows/browsers.yml +++ b/.github/workflows/browsers.yml @@ -1,7 +1,7 @@ name: Browsers on: - # - push + - push - workflow_dispatch jobs: diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 1f842aa5..9316a0e8 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -25,7 +25,7 @@ jobs: node-version: ${{ matrix.node-version }} - run: npm ci - run: npm test - # - run: npm run test:dist + - run: npm run test:dist lint: runs-on: ubuntu-latest From d1e59c5a8686a538413e9d6a98024a945df3554f Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Sun, 14 Feb 2021 00:23:19 +0200 Subject: [PATCH 89/89] Drop example/test files from src/parse/ --- src/parse/cst-stream.ts | 39 --------------------------------------- src/parse/test.ts | 18 ------------------ 2 files changed, 57 deletions(-) delete mode 100644 src/parse/cst-stream.ts delete mode 100644 src/parse/test.ts diff --git a/src/parse/cst-stream.ts b/src/parse/cst-stream.ts deleted file mode 100644 index 60542bcd..00000000 --- a/src/parse/cst-stream.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { Transform, TransformOptions } from 'stream' -import { StringDecoder } from 'string_decoder' -import { Parser } from './parser.js' - -export type ParseStreamOptions = Omit< - TransformOptions, - 'decodeStrings' | 'emitClose' | 'objectMode' -> - -export class CSTStream extends Transform { - decoder: StringDecoder - parser: Parser - - constructor(options: ParseStreamOptions = {}) { - super({ - ...options, - decodeStrings: false, - emitClose: true, - objectMode: true - }) - this.decoder = new StringDecoder(options.defaultEncoding || 'utf8') - this.parser = new Parser(token => this.push(token)) - } - - _flush(done: (error?: Error) => void) { - this.parser.parse('', false) - done() - } - - _transform(chunk: string | Buffer, _: any, done: (error?: Error) => void) { - try { - const src = Buffer.isBuffer(chunk) ? this.decoder.write(chunk) : chunk - this.parser.parse(src, true) - done() - } catch (error) { - done(error) // should never happen - } - } -} diff --git a/src/parse/test.ts b/src/parse/test.ts deleted file mode 100644 index 3a435a70..00000000 --- a/src/parse/test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { CSTStream } from './cst-stream.js' -import { Parser } from './parser.js' - -export function stream(source: string) { - const ps = new CSTStream().on('data', d => console.dir(d, { depth: null })) - ps.write(source) - ps.end() -} - -export function test(source: string) { - const lines: number[] = [] - const parser = new Parser( - t => console.dir(t, { depth: null }), - n => lines.push(n) - ) - parser.parse(source) - console.log({ lines }) -}