Merge pull request #201 from eemeli/re-resolve

Refactor tag resolve() API

docs/06_custom_tags.md

@@ -110,7 +110,7 @@ If you wish to implement your own custom tags, the [`!!binary`](https://github.c
 
 At the lowest level, [`YAML.parseCST()`](#cst-parser) will take care of turning string input into a concrete syntax tree (CST). In the CST all scalar values are available as strings, and maps & sequences as collections of nodes. Each schema includes a set of default data types, which handle converting at least strings, maps and sequences into their AST nodes. These are considered to have _implicit_ tags, and are autodetected. Custom tags, on the other hand, should almost always define an _explicit_ `tag` with which their value will be prefixed. This may be application-specific local `!tag`, a shorthand `!ns!tag`, or a verbatim `!<tag:example.com,2019:tag>`.
 
-Once identified by matching the `tag`, the `resolve(doc, cstNode): Node | any` function will turn a CST node into an AST node. For scalars, this is relatively simple, as the stringified node value is directly available, and should be converted to its actual value. Collections are trickier, and it's almost certain that it'll make sense to use the `parseMap(doc, cstNode)` and `parseSeq(doc, cstNode)` functions exported from `'yaml/util'` to initially resolve the CST collection into a `YAMLMap` or `YAMLSeq` object, and to work with that instead -- this is for instance what the YAML 1.1 collections do.
+Once identified by matching the `tag`, the `resolve(value, onError): Node | any` function will turn a parsed value into an AST node. `value` may be either a `string`, a `YAMLMap` or a `YAMLSeq`, depending on the node's shape. A custom tag should verify that value is of its expected type.
 
 Note that during the CST -> AST parsing, the anchors and comments attached to each node are also resolved for each node. This metadata will unfortunately be lost when converting the values to JS objects, so collections should have values that extend one of the existing collection classes. Collections should therefore either fall back to their parent classes' `toJSON()` methods, or define their own in order to allow their contents to be expressed as the appropriate JS object.
 
@@ -128,8 +128,6 @@ Finally, `stringify(item, ctx, ...): string` defines how your data should be rep
 ```js
 import {
   findPair, // (items, key) => Pair? -- Given a key, find a matching Pair
-  parseMap, // (doc, cstNode) => new YAMLMap
-  parseSeq, // (doc, cstNode) => new YAMLSeq
   stringifyNumber, // (node) => string
   stringifyString, // (node, ctx, ...) => string
   toJS, // (value, arg, ctx) => any -- Recursively convert to plain JS
@@ -145,7 +143,7 @@ To define your own tag, you'll need to define an object comprising of some of th
 - **`identify(value): boolean`** is used by `doc.createNode()` to detect your data type, e.g. using `typeof` or `instanceof`. Required.
 - `nodeClass: Node` is the `Node` child class that implements this tag. Required for collections and tags that have overlapping JS representations.
 - `options: Object` is used by some tags to configure their stringification.
-- **`resolve(doc, cstNode): Node | any`** turns a CST node into an AST node; `doc` is the resulting `YAML.Document` instance. If returning a non-`Node` value, the output will be wrapped as a `Scalar`. Required.
+- **`resolve(value, onError): Node | any`** turns a parsed value into an AST node; `value` is either a `string`, a `YAMLMap` or a `YAMLSeq`. `onError(msg)` should be called with an error message string when encountering errors, as it'll allow you to still return some value for the node. If returning a non-`Node` value, the output will be wrapped as a `Scalar`. Required.
 - `stringify(item, ctx, onComment, onChompKeep): string` is an optional function stringifying the `item` AST node in the current context `ctx`. `onComment` and `onChompKeep` are callback functions for a couple of special cases. If your data includes a suitable `.toString()` method, you can probably leave this undefined and use the default stringifier.
 - **`tag: string`** is the identifier for your data type, with which its stringified form will be prefixed. Should either be a !-prefixed local `!tag`, or a fully qualified `tag:domain,date:foo`. Required.
 - `test: RegExp` and `default: boolean` allow for values to be stringified without an explicit tag and detected using a regular expression. For most cases, it's unlikely that you'll actually want to use these, even if you first think you do.

src/resolve/resolveMap.js

@@ -16,11 +16,6 @@ import {
 import { resolveNode } from './resolveNode.js'
 
 export function resolveMap(doc, cst) {
-  if (cst.type !== Type.MAP && cst.type !== Type.FLOW_MAP) {
-    const msg = `A ${cst.type} node cannot be resolved as a mapping`
-    doc.errors.push(new YAMLSyntaxError(cst, msg))
-    return null
-  }
   const { comments, items } =
     cst.type === Type.FLOW_MAP
       ? resolveFlowMapItems(doc, cst)

src/resolve/resolveNode.js

@@ -7,7 +7,6 @@ import {
 } from '../errors.js'
 
 import { resolveScalar } from './resolveScalar.js'
-import { resolveString } from './resolveString.js'
 import { resolveTagName } from './resolveTagName.js'
 import { resolveTag } from './resolveTag.js'
 
@@ -93,8 +92,12 @@ function resolveNodeValue(doc, node) {
   }
 
   try {
-    const str = resolveString(doc, node)
-    return resolveScalar(str, schema.tags, schema.tags.scalarFallback)
+    let str = node.strValue || ''
+    if (typeof str !== 'string') {
+      str.errors.forEach(error => doc.errors.push(error))
+      str = str.str
+    }
+    return resolveScalar(str, schema.tags)
   } catch (error) {
     if (!error.source) error.source = node
     errors.push(error)

src/resolve/resolveScalar.js

@@ -1,18 +1,13 @@
 import { Scalar } from '../ast/Scalar.js'
 
-// falls back to string on no match
-export function resolveScalar(str, tags, scalarFallback) {
+export function resolveScalar(str, tags) {
   for (const { format, test, resolve } of tags) {
-    if (test) {
-      const match = str.match(test)
-      if (match) {
-        let res = resolve.apply(null, match)
-        if (!(res instanceof Scalar)) res = new Scalar(res)
-        if (format) res.format = format
-        return res
-      }
+    if (test && test.test(str)) {
+      let res = resolve(str)
+      if (!(res instanceof Scalar)) res = new Scalar(res)
+      if (format) res.format = format
+      return res
     }
   }
-  if (scalarFallback) str = scalarFallback(str)
-  return new Scalar(str)
+  return new Scalar(str) // fallback to string
 }

src/resolve/resolveSeq.js

@@ -13,11 +13,6 @@ import {
 import { resolveNode } from './resolveNode.js'
 
 export function resolveSeq(doc, cst) {
-  if (cst.type !== Type.SEQ && cst.type !== Type.FLOW_SEQ) {
-    const msg = `A ${cst.type} node cannot be resolved as a sequence`
-    doc.errors.push(new YAMLSyntaxError(cst, msg))
-    return null
-  }
   const { comments, items } =
     cst.type === Type.FLOW_SEQ
       ? resolveFlowSeqItems(doc, cst)

src/resolve/resolveTag.js

@@ -1,53 +1,73 @@
 import { Collection } from '../ast/Collection.js'
 import { Scalar } from '../ast/Scalar.js'
 import { Type, defaultTags } from '../constants.js'
-import { YAMLReferenceError, YAMLWarning } from '../errors.js'
+import {
+  YAMLReferenceError,
+  YAMLSemanticError,
+  YAMLWarning
+} from '../errors.js'
+import { resolveMap } from './resolveMap.js'
 import { resolveScalar } from './resolveScalar.js'
-import { resolveString } from './resolveString.js'
+import { resolveSeq } from './resolveSeq.js'
 
-function resolveByTagName(doc, node, tagName) {
-  const { knownTags, tags } = doc.schema
+function resolveByTagName({ knownTags, tags }, tagName, value, onError) {
   const matchWithTest = []
   for (const tag of tags) {
     if (tag.tag === tagName) {
-      if (tag.test) matchWithTest.push(tag)
-      else {
-        const res = tag.resolve(doc, node)
+      if (tag.test) {
+        if (typeof value === 'string') matchWithTest.push(tag)
+        else onError(`The tag ${tagName} cannot be applied to a collection`)
+      } else {
+        const res = tag.resolve(value, onError)
         return res instanceof Collection ? res : new Scalar(res)
       }
     }
   }
-
-  const str = resolveString(doc, node)
-  if (typeof str === 'string' && matchWithTest.length > 0)
-    return resolveScalar(str, matchWithTest, tags.scalarFallback)
+  if (matchWithTest.length > 0) return resolveScalar(value, matchWithTest)
 
   const kt = knownTags[tagName]
   if (kt) {
     tags.push(Object.assign({}, kt, { default: false, test: undefined }))
-    const res = kt.resolve(doc, node)
+    const res = kt.resolve(value, onError)
     return res instanceof Collection ? res : new Scalar(res)
   }
 
   return null
 }
 
-function getFallbackTagName({ type }) {
-  switch (type) {
-    case Type.FLOW_MAP:
-    case Type.MAP:
-      return defaultTags.MAP
-    case Type.FLOW_SEQ:
-    case Type.SEQ:
-      return defaultTags.SEQ
-    default:
-      return defaultTags.STR
-  }
-}
-
 export function resolveTag(doc, node, tagName) {
+  const { MAP, SEQ, STR } = defaultTags
+  let value, fallback
+  const onError = message =>
+    doc.errors.push(new YAMLSemanticError(node, message))
   try {
-    const res = resolveByTagName(doc, node, tagName)
+    switch (node.type) {
+      case Type.FLOW_MAP:
+      case Type.MAP:
+        value = resolveMap(doc, node)
+        fallback = MAP
+        if (tagName === SEQ || tagName === STR)
+          onError(`The tag ${tagName} cannot be applied to a mapping`)
+        break
+      case Type.FLOW_SEQ:
+      case Type.SEQ:
+        value = resolveSeq(doc, node)
+        fallback = SEQ
+        if (tagName === MAP || tagName === STR)
+          onError(`The tag ${tagName} cannot be applied to a sequence`)
+        break
+      default:
+        value = node.strValue || ''
+        if (typeof value !== 'string') {
+          value.errors.forEach(error => doc.errors.push(error))
+          value = value.str
+        }
+        if (tagName === MAP || tagName === SEQ)
+          onError(`The tag ${tagName} cannot be applied to a scalar`)
+        fallback = STR
+    }
+
+    const res = resolveByTagName(doc.schema, tagName, value, onError)
     if (res) {
       if (tagName && node.tag) res.tag = tagName
       return res
@@ -60,11 +80,10 @@ export function resolveTag(doc, node, tagName) {
   }
 
   try {
-    const fallback = getFallbackTagName(node)
     if (!fallback) throw new Error(`The tag ${tagName} is unavailable`)
     const msg = `The tag ${tagName} is unavailable, falling back to ${fallback}`
     doc.warnings.push(new YAMLWarning(node, msg))
-    const res = resolveByTagName(doc, node, fallback)
+    const res = resolveByTagName(doc.schema, fallback, value, onError)
     res.tag = tagName
     return res
   } catch (error) {

src/stringify/stringifyString.js

@@ -260,7 +260,7 @@ function plainString(item, ctx, onComment, onChompKeep) {
   // and others in v1.1.
   if (actualString) {
     const { tags } = ctx.doc.schema
-    const resolved = resolveScalar(str, tags, tags.scalarFallback).value
+    const resolved = resolveScalar(str, tags).value
     if (typeof resolved !== 'string') return doubleQuotedString(value, ctx)
   }
   const body = implicitKey

src/tags/core.js

@@ -8,8 +8,8 @@ import { boolOptions, intOptions, nullOptions } from './options.js'
 const intIdentify = value =>
   typeof value === 'bigint' || Number.isInteger(value)
 
-const intResolve = (src, part, radix) =>
-  intOptions.asBigInt ? BigInt(src) : parseInt(part, radix)
+const intResolve = (src, offset, radix) =>
+  intOptions.asBigInt ? BigInt(src) : parseInt(src.substring(offset), radix)
 
 function intStringify(node, radix, prefix) {
   const { value } = node
@@ -48,8 +48,8 @@ export const octObj = {
   default: true,
   tag: 'tag:yaml.org,2002:int',
   format: 'OCT',
-  test: /^0o([0-7]+)$/,
-  resolve: (str, oct) => intResolve(str, oct, 8),
+  test: /^0o[0-7]+$/,
+  resolve: str => intResolve(str, 2, 8),
   options: intOptions,
   stringify: node => intStringify(node, 8, '0o')
 }
@@ -59,7 +59,7 @@ export const intObj = {
   default: true,
   tag: 'tag:yaml.org,2002:int',
   test: /^[-+]?[0-9]+$/,
-  resolve: str => intResolve(str, str, 10),
+  resolve: str => intResolve(str, 0, 10),
   options: intOptions,
   stringify: stringifyNumber
 }
@@ -69,8 +69,8 @@ export const hexObj = {
   default: true,
   tag: 'tag:yaml.org,2002:int',
   format: 'HEX',
-  test: /^0x([0-9a-fA-F]+)$/,
-  resolve: (str, hex) => intResolve(str, hex, 16),
+  test: /^0x[0-9a-fA-F]+$/,
+  resolve: str => intResolve(str, 2, 16),
   options: intOptions,
   stringify: node => intStringify(node, 16, '0x')
 }
@@ -79,9 +79,9 @@ export const nanObj = {
   identify: value => typeof value === 'number',
   default: true,
   tag: 'tag:yaml.org,2002:float',
-  test: /^(?:[-+]?\.inf|(\.nan))$/i,
-  resolve: (str, nan) =>
-    nan
+  test: /^(?:[-+]?\.(?:inf|Inf|INF|nan|NaN|NAN))$/,
+  resolve: str =>
+    str.slice(-3).toLowerCase() === 'nan'
       ? NaN
       : str[0] === '-'
       ? Number.NEGATIVE_INFINITY
@@ -103,12 +103,12 @@ export const floatObj = {
   identify: value => typeof value === 'number',
   default: true,
   tag: 'tag:yaml.org,2002:float',
-  test: /^[-+]?(?:\.([0-9]+)|[0-9]+\.([0-9]*))$/,
-  resolve(str, frac1, frac2) {
-    const frac = frac1 || frac2
+  test: /^[-+]?(?:\.[0-9]+|[0-9]+\.[0-9]*)$/,
+  resolve(str) {
     const node = new Scalar(parseFloat(str))
-    if (frac && frac[frac.length - 1] === '0')
-      node.minFractionDigits = frac.length
+    const dot = str.indexOf('.')
+    if (dot !== -1 && str[str.length - 1] === '0')
+      node.minFractionDigits = str.length - dot - 1
     return node
   },
   stringify: stringifyNumber

src/tags/failsafe/map.js

@@ -1,6 +1,5 @@
 import { createPair } from '../../ast/Pair.js'
 import { YAMLMap } from '../../ast/YAMLMap.js'
-import { resolveMap } from '../../resolve/resolveMap.js'
 
 function createMap(schema, obj, ctx) {
   const { keepUndefined, replacer } = ctx
@@ -27,5 +26,5 @@ export const map = {
   default: true,
   nodeClass: YAMLMap,
   tag: 'tag:yaml.org,2002:map',
-  resolve: resolveMap
+  resolve: map => map
 }

src/tags/failsafe/seq.js

@@ -1,6 +1,5 @@
 import { YAMLSeq } from '../../ast/YAMLSeq.js'
 import { createNode } from '../../doc/createNode.js'
-import { resolveSeq } from '../../resolve/resolveSeq.js'
 
 function createSeq(schema, obj, ctx) {
   const { replacer } = ctx
@@ -23,5 +22,5 @@ export const seq = {
   default: true,
   nodeClass: YAMLSeq,
   tag: 'tag:yaml.org,2002:seq',
-  resolve: resolveSeq
+  resolve: seq => seq
 }

src/tags/failsafe/string.js

@@ -1,12 +1,11 @@
-import { resolveString } from '../../resolve/resolveString.js'
 import { stringifyString } from '../../stringify/stringifyString.js'
 import { strOptions } from '../options.js'
 
 export const string = {
   identify: value => typeof value === 'string',
   default: true,
   tag: 'tag:yaml.org,2002:str',
-  resolve: resolveString,
+  resolve: str => str,
   stringify(item, ctx, onComment, onChompKeep) {
     ctx = Object.assign({ actualString: true }, ctx)
     return stringifyString(item, ctx, onComment, onChompKeep)

src/tags/json.js

@@ -1,7 +1,6 @@
 /* global BigInt */
 
 import { Scalar } from '../ast/Scalar.js'
-import { resolveString } from '../resolve/resolveString.js'
 import { map } from './failsafe/map.js'
 import { seq } from './failsafe/seq.js'
 import { intOptions } from './options.js'
@@ -18,7 +17,7 @@ export const json = [
     identify: value => typeof value === 'string',
     default: true,
     tag: 'tag:yaml.org,2002:str',
-    resolve: resolveString,
+    resolve: str => str,
     stringify: stringifyJSON
   },
   {
@@ -55,9 +54,13 @@ export const json = [
     test: /^-?(?:0|[1-9][0-9]*)(?:\.[0-9]*)?(?:[eE][-+]?[0-9]+)?$/,
     resolve: str => parseFloat(str),
     stringify: stringifyJSON
+  },
+  {
+    default: true,
+    test: /^/,
+    resolve(str, onError) {
+      onError(`Unresolved plain scalar ${JSON.stringify(str)}`)
+      return str
+    }
   }
 ]
-
-json.scalarFallback = str => {
-  throw new SyntaxError(`Unresolved plain scalar ${JSON.stringify(str)}`)
-}