feat: Support default arguments referencing other arguments

Now expandMacros can correctly expand macros whose default arguments
reference other arguments. A special care was taken in order to follow
the behavior of xparse in case of circular references. In most of the
cases, xparse throws a compilation error, but it works in case like
`O{#2} O{#1}`. From this, we can reasonably guess that xparse treats
default arguments that directly copies another argument's value in a
special way. This behavior was implemented with a simple algorithm, and
it turns out to be consistent with xparse's behavior in cases considered
in unit tests.

packages/unified-latex-builder/libs/builders.ts

@@ -181,6 +181,16 @@ export function arg(
     return { type: "argument", content: args, openMark, closeMark };
 }
 
+/**
+ * Creates an empty argument. This can only present in Ast as a result of -NoValue-.
+ */
+export function emptyArg() {
+    return arg([], {
+        openMark: "",
+        closeMark: "",
+    });
+}
+
 /**
  * Create a Macro with the given `name`. The macro
  * may be followed by any number of arguments.

packages/unified-latex-prettier/libs/printer/argument.ts

@@ -19,11 +19,7 @@ export function printArgument(
     );
 
     // We can return early for empty arguments (this is common for omitted optional arguments)
-    if (
-        node.openMark === "" &&
-        node.closeMark === "" &&
-        node.content.length === 0
-    ) {
+    if (match.blankArgument(node)) {
         return [];
     }
     const parentNode = path.getParentNode();

packages/unified-latex-prettier/libs/printer/printer.ts

@@ -83,6 +83,8 @@ export function printLatexAst(
             return printVerbatimEnvironment(path, print, options);
         case "whitespace":
             return line;
+        case "hash_number":
+            return "#" + node.number.toString(10);
         default:
             console.warn("Printing unknown type", node);
             return printRaw(node);

packages/unified-latex-to-hast/libs/html-subs/to-hast-direct.ts

@@ -69,6 +69,11 @@ export function toHastDirect(
             );
         case "root":
             return h("root");
+        case "hash_number":
+            return {
+                type: "text",
+                value: `#${node.number}`,
+            };
         default: {
             const _exhaustiveCheck: never = node;
             throw new Error(

packages/unified-latex-to-hast/libs/html-subs/to-hast.ts

@@ -114,6 +114,11 @@ export function toHastWithLoggerFactory(
                 );
             case "root":
                 return node.content.flatMap(toHast);
+            case "hash_number":
+                return {
+                    type: "text",
+                    value: `#${node.number}`,
+                };
             default: {
                 const _exhaustiveCheck: never = node;
                 throw new Error(

packages/unified-latex-types/libs/ast-types.ts

@@ -92,6 +92,12 @@ export interface Argument extends ContentNode {
     closeMark: string;
 }
 
+// Only available during macro expansion
+export interface HashNumber extends BaseNode {
+    type: "hash_number";
+    number: number;
+}
+
 export type Node =
     | Root
     | String
@@ -104,6 +110,7 @@ export type Node =
     | InlineMath
     | DisplayMath
     | Group
-    | Verb;
+    | Verb
+    | HashNumber;
 
 export type Ast = Node | Argument | Node[];

packages/unified-latex-util-arguments/libs/get-args-content.ts

@@ -1,5 +1,5 @@
 import * as Ast from "@unified-latex/unified-latex-types";
-
+import { blankArgument } from "@unified-latex/unified-latex-util-match";
 /**
  * Returns the content of `args` for a macro or environment as an array. If an argument
  * was omitted (e.g., because it was an optional arg that wasn't included), then `null` is returned.
@@ -12,7 +12,7 @@ export function getArgsContent(
     }
 
     return node.args.map((arg) => {
-        if (arg.openMark === "" && arg.content.length === 0) {
+        if (blankArgument(arg)) {
             return null;
         }
         return arg.content;
@@ -48,7 +48,7 @@ export function getNamedArgsContent(
             return;
         }
         let val: Ast.Node[] | null = arg.content;
-        if (arg.openMark === "" && arg.content.length === 0) {
+        if (blankArgument(arg)) {
             val = null;
         }
         ret[name] = val;

packages/unified-latex-util-arguments/libs/gobble-arguments.ts

@@ -1,10 +1,11 @@
+import { emptyArg } from "@unified-latex/unified-latex-builder";
 import * as Ast from "@unified-latex/unified-latex-types";
 import { ArgumentParser } from "@unified-latex/unified-latex-types";
 import {
     ArgSpecAst as ArgSpec,
     parse as parseArgspec,
 } from "@unified-latex/unified-latex-util-argspec";
-import { emptyArg, gobbleSingleArgument } from "./gobble-single-argument";
+import { gobbleSingleArgument } from "./gobble-single-argument";
 
 /**
  * Gobbles an argument of whose type is specified

packages/unified-latex-util-arguments/libs/gobble-single-argument.ts

@@ -353,10 +353,3 @@ function parseToken(
     }
     return str;
 }
-
-/**
- * Create an empty argument.
- */
-export function emptyArg(): Ast.Argument {
-    return arg([], { openMark: "", closeMark: "" });
-}

packages/unified-latex-util-macros/libs/newcommand.ts

@@ -1,16 +1,14 @@
 import { structuredClone } from "@unified-latex/structured-clone";
+import { emptyArg, s } from "@unified-latex/unified-latex-builder";
 import * as Ast from "@unified-latex/unified-latex-types";
+import { parse as parseArgspec } from "@unified-latex/unified-latex-util-argspec";
+import { getNamedArgsContent } from "@unified-latex/unified-latex-util-arguments";
+import * as match from "@unified-latex/unified-latex-util-match";
+import { parse } from "@unified-latex/unified-latex-util-parse";
 import { printRaw } from "@unified-latex/unified-latex-util-print-raw";
 import { replaceNode } from "@unified-latex/unified-latex-util-replace";
 import { visit } from "@unified-latex/unified-latex-util-visit";
-import { getNamedArgsContent } from "@unified-latex/unified-latex-util-arguments";
-import { parse as parseArgspec } from "@unified-latex/unified-latex-util-argspec";
-import { s } from "@unified-latex/unified-latex-builder";
-import { parse } from "@unified-latex/unified-latex-util-parse";
-import {
-    HashNumber,
-    parseMacroSubstitutions,
-} from "./parse-macro-substitutions";
+import { parseMacroSubstitutions } from "./parse-macro-substitutions";
 
 export const LATEX_NEWCOMMAND = new Set([
     "newcommand",
@@ -194,24 +192,9 @@ export function createMacroExpander(
     signature?: string
 ): (macro: Ast.Macro) => Ast.Node[] {
     const cachedSubstitutionTree = structuredClone(substitution);
-    let hasSubstitutions = false;
-    visit(
-        cachedSubstitutionTree,
-        (nodes) => {
-            const parsed = parseMacroSubstitutions(nodes);
-            // Keep track of whether there are any substitutions so we can bail early if not.
-            hasSubstitutions =
-                hasSubstitutions ||
-                parsed.some((node) => node.type === "hash_number");
-            nodes.length = 0;
-            nodes.push(...(parsed as Ast.Node[]));
-        },
-        {
-            includeArrays: true,
-            test: Array.isArray,
-        }
-    );
-    if (!hasSubstitutions) {
+    const res = getMacroSubstitutionHashNumbers(cachedSubstitutionTree);
+
+    if (res.size === 0) {
         return () => structuredClone(cachedSubstitutionTree);
     }
 
@@ -232,36 +215,111 @@ export function createMacroExpander(
 
     return (macro: Ast.Macro) => {
         const retTree = structuredClone(cachedSubstitutionTree);
-        replaceNode(retTree, (node) => {
-            const hashNumOrNode = node as Ast.Node | HashNumber;
-            if (hashNumOrNode.type !== "hash_number") {
-                return;
-            }
+        const args = macro.args;
 
-            const arg = macro.args?.[hashNumOrNode.number - 1];
+        const stack: number[] = [];
+        let lastSelfReference: number | null = null;
 
-            // Check if this argument is -NoValue-
-            if (
-                !arg ||
-                (arg.content.length === 0 &&
-                    arg.openMark === "" &&
-                    arg.closeMark === "")
-            ) {
-                // Check if there exists a default argument for this hash number
-                const defaultArg = defaultArgs[hashNumOrNode.number - 1];
-                if (!defaultArg) {
-                    return s(`#${hashNumOrNode.number}`);
+        // Recursively expand macro arguments. If a self-reference is found, it returns
+        // the corresponding hash number, which is used to special-case `O{#2} O{#1}`.
+        function expandArgs(retTree: Ast.Node[]): void {
+            replaceNode(retTree, (node) => {
+                if (node.type !== "hash_number") {
+                    return;
                 }
-                // `defaultArg` is a string expression. The same `defaultArg` may be parsed
-                // differently depending on the context of `macro`, so we cannot cache
-                // the parse result of `defaultArg`. FIXME: we should probably pass some options
-                // that is provided to whatever function that called this to the below parse call.
-                const root = parse(defaultArg);
-                return root.content;
-            }
 
-            return arg.content;
-        });
+                const hashNum = node.number;
+                const arg = args?.[hashNum - 1];
+
+                // Check if this argument is -NoValue-
+                if (!arg || match.blankArgument(arg)) {
+                    // Check if there exists a default argument for this hash number
+                    const defaultArg = defaultArgs[hashNum - 1];
+                    if (!defaultArg) {
+                        return s(`#${hashNum}`);
+                    }
+
+                    // Detect self-references
+                    if (stack.includes(hashNum)) {
+                        lastSelfReference = hashNum;
+                        return s(`#${hashNum}`);
+                    }
+
+                    // `defaultArg` is a string expression. The same `defaultArg` may be parsed
+                    // differently depending on the context of `macro`, so we cannot cache
+                    // the parse result of `defaultArg`. Currently we just call `parse` without
+                    // taking account of parsing contexts, so actually the result can be cached,
+                    // but this is not the correct thing to do. FIXME: we should probably pass
+                    // some options that is provided to whatever function that called this to
+                    // the below parse call. Note that `parse` is done in several passes, and we
+                    // may be able to cache result of a first few passes that aren't context-dependent.
+                    const subst = parse(defaultArg).content;
+                    const nextHashNums = getMacroSubstitutionHashNumbers(subst);
+
+                    if (nextHashNums.size === 0) {
+                        return subst;
+                    }
+
+                    stack.push(hashNum);
+                    try {
+                        expandArgs(subst);
+
+                        if (lastSelfReference !== hashNum) {
+                            return subst;
+                        }
+
+                        // At this point, we have encountered #n while expanding #n.
+                        // Check if we got exactly #n by expanding #n,
+                        // in which case we should return the -NoValue-.
+                        if (`#${hashNum}` === printRaw(subst)) {
+                            // We are good, clear the last self-reference variable
+                            lastSelfReference = null;
+                            return emptyArg();
+                        }
+
+                        console.warn(
+                            `Detected unrecoverable self-reference while expanding macro: ${printRaw(
+                                macro
+                            )}`
+                        );
+                        // Return a placeholder string, so that we know that
+                        // this code path is not taken in unit tests.
+                        return s("-Circular-");
+                    } finally {
+                        stack.pop();
+                    }
+                }
+
+                return arg.content;
+            });
+        }
+
+        expandArgs(retTree);
         return retTree;
     };
 }
+
+/**
+ * Parses macro substitutions, mutates tree, and returns a list of hashnumbers that were encountered.
+ */
+export function getMacroSubstitutionHashNumbers(tree: Ast.Node[]) {
+    const hashNumbers = new Set<number>();
+    visit(
+        tree,
+        (nodes) => {
+            const parsed = parseMacroSubstitutions(nodes);
+            parsed.forEach((node) => {
+                if (node.type === "hash_number") {
+                    hashNumbers.add(node.number);
+                }
+            });
+            nodes.length = 0;
+            nodes.push(...(parsed as Ast.Node[]));
+        },
+        {
+            includeArrays: true,
+            test: Array.isArray,
+        }
+    );
+    return hashNumbers;
+}

packages/unified-latex-util-macros/libs/parse-macro-substitutions.ts

@@ -1,14 +1,9 @@
-import { MacroSubstitutionPegParser } from "@unified-latex/unified-latex-util-pegjs";
 import * as Ast from "@unified-latex/unified-latex-types";
 import { match } from "@unified-latex/unified-latex-util-match";
-import { decorateArrayForPegjs } from "@unified-latex/unified-latex-util-pegjs";
-
-// The types returned by the grammar
-
-export interface HashNumber {
-    type: "hash_number";
-    number: number;
-}
+import {
+    MacroSubstitutionPegParser,
+    decorateArrayForPegjs,
+} from "@unified-latex/unified-latex-util-pegjs";
 
 export function createMatchers() {
     return {
@@ -35,9 +30,7 @@ export function createMatchers() {
  *
  * The resulting AST is ready for substitutions to be applied to it.
  */
-export function parseMacroSubstitutions(
-    ast: Ast.Node[]
-): (Ast.Node | HashNumber)[] {
+export function parseMacroSubstitutions(ast: Ast.Node[]): Ast.Node[] {
     if (!Array.isArray(ast)) {
         throw new Error("You must pass an array of nodes");
     }