implement indexing + querying over canvas files

.github/FUNDING.yml

@@ -1 +1 @@
-github: ["blacksmithgu"]
+github: ["blacksmithgu"];

__mocks__/data-import/web-worker/import-manager.ts

@@ -9,6 +9,7 @@ export class FileImporter {
     public async reload<T>(file: TFile): Promise<T> {
         let contents = await this.vault.read(file);
         let metadata = await this.metadataCache.getFileCache(file);
-        return runImport(file.path, contents, file.stat, metadata as CachedMetadata) as any as T;
+        // @ts-ignore it exists!!!! please stop bothering me.
+        return runImport(file.path, contents, file.stat, metadata as CachedMetadata, window.app.fileManager.linkUpdaters.canvas.canvas.index.index) as any as T;
     }
 }

src/api/plugin-api.ts

@@ -34,6 +34,7 @@ import { Query } from "query/query";
 import { DataviewCalendarRenderer } from "ui/views/calendar-view";
 import { DataviewJSRenderer } from "ui/views/js-view";
 import { markdownList, markdownTable, markdownTaskList } from "ui/export/markdown";
+import { SCanvas } from "data-model/serialized/canvas";
 
 /** Asynchronous API calls related to file / system IO. */
 export class DataviewIOApi {
@@ -158,7 +159,7 @@ export class DataviewApi {
     }
 
     /** Remaps important metadata to add data arrays.  */
-    private _addDataArrays(pageObject: SMarkdownPage): SMarkdownPage {
+    private _addDataArrays(pageObject: SMarkdownPage | SCanvas): SMarkdownPage | SCanvas {
         // Remap the "file" metadata entries to be data arrays.
         for (let [key, value] of Object.entries(pageObject.file)) {
             if (Array.isArray(value)) (pageObject.file as any)[key] = DataArray.wrap<any>(value, this.settings);

src/data-import/canvas-file.ts

@@ -0,0 +1,25 @@
+/** importer for canvas nodes */
+
+// import { CanvasMetadata } from "data-model/canvas";
+import { CanvasCard, CanvasMetadataIndex } from "data-model/canvas";
+import { FileStats /* parseYaml */ } from "obsidian";
+import { parsePage } from "./markdown-file";
+
+export function parseCanvasCard(
+    path: string,
+    id: string,
+    contents: string,
+    stat: FileStats,
+    mindex: CanvasMetadataIndex
+) {
+    // @ts-expect-error SHUT UP MEG
+    const metadata = mindex[path]?.caches[id];
+
+    let data = JSON.parse(contents);
+    // @ts-ignore
+    // data.nodes = data.nodes
+    let current = data.nodes[data.nodes.findIndex((a: any) => a.id === id)];
+    const parsedPage = parsePage(path, current.text, stat, metadata);
+
+    return new CanvasCard(current, path, stat, parsedPage);
+}

src/data-import/markdown-file.ts

@@ -63,7 +63,6 @@ export function parsePage(path: string, contents: string, stat: FileStats, metad
     // Strip "position" from frontmatter since it is Obsidian determined.
     const frontmatter = metadata.frontmatter || ({} as Record<string, any>);
     if (frontmatter && "position" in frontmatter) delete frontmatter["position"];
-
     return new PageMetadata(path, {
         tags,
         aliases,

src/data-import/persister.ts

@@ -1,3 +1,4 @@
+import { CanvasMetadata } from "data-model/canvas";
 import { PageMetadata } from "data-model/markdown";
 import { Transferable } from "data-model/transferable";
 import localforage from "localforage";
@@ -36,16 +37,16 @@ export class LocalStorageCache {
     }
 
     /** Load file metadata by path. */
-    public async loadFile(path: string): Promise<Cached<Partial<PageMetadata>> | null | undefined> {
+    public async loadFile(path: string): Promise<Cached<Partial<PageMetadata> | CanvasMetadata> | null | undefined> {
         return this.persister.getItem(this.fileKey(path)).then(raw => {
-            let result = raw as any as Cached<Partial<PageMetadata>>;
+            let result = raw as any as Cached<Partial<PageMetadata> | CanvasMetadata>;
             if (result) result.data = Transferable.value(result.data);
             return result;
         });
     }
 
     /** Store file metadata by path. */
-    public async storeFile(path: string, data: Partial<PageMetadata>): Promise<void> {
+    public async storeFile(path: string, data: Partial<PageMetadata | CanvasMetadata>): Promise<void> {
         await this.persister.setItem(this.fileKey(path), {
             version: this.version,
             time: Date.now(),

src/data-import/web-worker/import-entry.ts

@@ -1,21 +1,28 @@
 /** Entry-point script used by the index as a web worker. */
 import { runImport } from "data-import/web-worker/import-impl";
+import { CanvasMetadataIndex } from "data-model/canvas";
 import { Transferable } from "data-model/transferable";
 import { CachedMetadata, FileStats } from "obsidian";
 
 /** An import which can fail and raise an exception, which will be caught by the handler. */
-function failableImport(path: string, contents: string, stat: FileStats, metadata?: CachedMetadata) {
+function failableImport(
+    path: string,
+    contents: string,
+    stat: FileStats,
+    mindex: CanvasMetadataIndex,
+    metadata?: CachedMetadata
+) {
     if (metadata === undefined || metadata === null) {
         throw Error(`Cannot index file, since it has no Obsidian file metadata.`);
     }
 
-    return runImport(path, contents, stat, metadata);
+    return runImport(path, contents, stat, metadata, mindex);
 }
 
 onmessage = async evt => {
     try {
-        let { path, contents, stat, metadata } = evt.data;
-        let result = failableImport(path, contents, stat, metadata);
+        let { path, contents, stat, metadata, mindex } = evt.data;
+        let result = failableImport(path, contents, stat, mindex, metadata);
         (postMessage as any)({ path: evt.data.path, result: Transferable.transferable(result) });
     } catch (error) {
         console.log(error);

src/data-import/web-worker/import-impl.ts

@@ -1,13 +1,30 @@
 /** Actual import implementation backend. This must remain separate from `import-entry` since it is used without web workers. */
+import { parseCanvasCard } from "data-import/canvas-file";
 import { parsePage } from "data-import/markdown-file";
+import { CanvasMetadata, CanvasMetadataIndex } from "data-model/canvas";
 import { PageMetadata } from "data-model/markdown";
 import { CachedMetadata, FileStats } from "obsidian";
 
+// TODO: array.isArray()...
 export function runImport(
     path: string,
     contents: string,
     stats: FileStats,
-    metadata: CachedMetadata
-): Partial<PageMetadata> {
+    metadata: CachedMetadata,
+    mindex: CanvasMetadataIndex
+): Partial<PageMetadata | CanvasMetadata> {
+    if (path.endsWith(".canvas")) {
+        const data = JSON.parse(contents);
+        const cm = new CanvasMetadata(
+            path,
+            data.nodes
+                .filter((a: any) => a.type === "text")
+                .map((a: any) => {
+                    return parseCanvasCard(path, a.id, contents, stats, mindex);
+                }),
+            stats
+        );
+        return cm;
+    }
     return parsePage(path, contents, stats, metadata);
 }

src/data-import/web-worker/import-manager.ts

@@ -2,7 +2,7 @@
 
 import { Transferable } from "data-model/transferable";
 import DataviewImportWorker from "web-worker:./import-entry.ts";
-import { Component, MetadataCache, TFile, Vault } from "obsidian";
+import { App, Component, MetadataCache, TFile, Vault } from "obsidian";
 
 /** Callback when a file is resolved. */
 type FileCallback = (p: any) => void;
@@ -21,7 +21,12 @@ export class FileImporter extends Component {
     /** Paths -> promises for file reloads which have not yet been queued. */
     callbacks: Map<string, [FileCallback, FileCallback][]>;
 
-    public constructor(public numWorkers: number, public vault: Vault, public metadataCache: MetadataCache) {
+    public constructor(
+        public numWorkers: number,
+        public vault: Vault,
+        public metadataCache: MetadataCache,
+        public app: App
+    ) {
         super();
         this.workers = [];
         this.busy = [];
@@ -93,14 +98,32 @@ export class FileImporter extends Component {
     private send(file: TFile, workerId: number) {
         this.busy[workerId] = true;
 
-        this.vault.cachedRead(file).then(c =>
-            this.workers[workerId].postMessage({
-                path: file.path,
-                contents: c,
-                stat: file.stat,
-                metadata: this.metadataCache.getFileCache(file),
-            })
-        );
+        this.vault.cachedRead(file).then(c => {
+            if (file.path.endsWith(".canvas")) {
+                const data = JSON.parse(c);
+                return data.nodes
+                    .filter((a: any) => a.type === "text")
+                    .forEach((b: any) => {
+                        this.workers[workerId].postMessage({
+                            path: file.path,
+                            contents: c,
+                            stat: file.stat,
+                            metadata:
+                                // @ts-expect-error SHUT UP MEG
+                                this.app.fileManager.linkUpdaters.canvas.canvas.index.index[file.path].caches[b.id],
+                            // @ts-expect-error SHUT UP MEG
+                            mindex: this.app.fileManager.linkUpdaters.canvas.canvas.index.index,
+                        });
+                    });
+            } else {
+                return this.workers[workerId].postMessage({
+                    path: file.path,
+                    contents: c,
+                    stat: file.stat,
+                    metadata: this.metadataCache.getFileCache(file),
+                });
+            }
+        });
     }
 
     /** Find the next available, non-busy worker; return undefined if all workers are busy. */

src/data-index/index.ts

@@ -3,6 +3,7 @@ import { Result } from "api/result";
 import { parseCsv } from "data-import/csv";
 import { LocalStorageCache } from "data-import/persister";
 import { FileImporter } from "data-import/web-worker/import-manager";
+import { CanvasCard, CanvasMetadata } from "data-model/canvas";
 import { PageMetadata } from "data-model/markdown";
 import { DataObject } from "data-model/value";
 import { DateTime } from "luxon";
@@ -27,7 +28,7 @@ export class FullIndex extends Component {
     public persister: LocalStorageCache;
 
     /* Maps path -> markdown metadata for all markdown pages. */
-    public pages: Map<string, PageMetadata>;
+    public pages: Map<string, PageMetadata | CanvasMetadata>;
 
     /** Map files -> tags in that file, and tags -> files. This version includes subtags. */
     public tags: ValueCaseInsensitiveIndexMap;
@@ -71,13 +72,23 @@ export class FullIndex extends Component {
         this.persister = new LocalStorageCache(app.appId || "shared", indexVersion);
 
         // Handles asynchronous reloading of files on web workers.
-        this.addChild((this.importer = new FileImporter(2, this.vault, this.metadataCache)));
+        this.addChild((this.importer = new FileImporter(2, this.vault, this.metadataCache, app)));
         // Prefix listens to file creation/deletion/rename, and not modifies, so we let it set up it's own listeners.
         this.addChild((this.prefix = PrefixIndex.create(this.vault, () => this.touch())));
         // The CSV cache also needs to listen to filesystem events for cache invalidation.
         this.addChild((this.csv = new CsvCache(this.vault)));
         // The starred cache fetches starred entries semi-regularly via an interval.
         this.addChild((this.starred = new StarredCache(this.app, () => this.touch())));
+        /** this is needed because though canvas files will also fire the `modify` event,
+            dataview can't already pick up on that. so we need to add the hook manually for canvases.
+         */
+        this.registerEvent(
+            this.vault.on("modify", async file => {
+                if (file instanceof TFile && PathFilters.canvas(file.path)) {
+                    await this.reload(file);
+                }
+            })
+        );
     }
 
     /** Trigger a metadata event on the metadata cache. */
@@ -102,7 +113,7 @@ export class FullIndex extends Component {
         // File creation does cause a metadata change, but deletes do not. Clear the caches for this.
         this.registerEvent(
             this.vault.on("delete", af => {
-                if (!(af instanceof TFile) || !PathFilters.markdown(af.path)) return;
+                if (!(af instanceof TFile) || (!PathFilters.markdown(af.path) && !PathFilters.canvas(af.path))) return;
                 let file = af as TFile;
 
                 this.pages.delete(file.path);
@@ -116,14 +127,14 @@ export class FullIndex extends Component {
         );
 
         // Asynchronously initialize actual content in the background.
-        this._initialize(this.vault.getMarkdownFiles());
+        this._initialize(this.vault.getFiles().filter(a => a.extension === "md" || a.extension === "canvas"));
     }
 
     /** Drops the local storage cache and re-indexes all files; this should generally be used if you expect cache issues. */
     public async reinitialize() {
         await this.persister.recreate();
 
-        const files = this.vault.getMarkdownFiles();
+        const files = this.vault.getFiles().filter(a => a.extension === "md" || a.extension === "canvas");
         const start = Date.now();
         let promises = files.map(file => this.reload(file));
 
@@ -164,7 +175,7 @@ export class FullIndex extends Component {
     }
 
     public rename(file: TAbstractFile, oldPath: string) {
-        if (!(file instanceof TFile) || !PathFilters.markdown(file.path)) return;
+        if (!(file instanceof TFile) || (!PathFilters.markdown(file.path) && !PathFilters.canvas(file.path))) return;
 
         if (this.pages.has(oldPath)) {
             const oldMeta = this.pages.get(oldPath);
@@ -185,7 +196,7 @@ export class FullIndex extends Component {
 
     /** Queue a file for reloading; this is done asynchronously in the background and may take a few seconds. */
     public async reload(file: TFile): Promise<{ cached: boolean; skipped: boolean }> {
-        if (!PathFilters.markdown(file.path)) return { cached: false, skipped: true };
+        if (!PathFilters.markdown(file.path) && !PathFilters.canvas(file.path)) return { cached: false, skipped: true };
 
         // The first load of a file is attempted from persisted cache; subsequent loads just use the importer.
         if (this.pages.has(file.path) || this.initialized) {
@@ -214,24 +225,37 @@ export class FullIndex extends Component {
 
     /** Import a file directly from disk, skipping the cache. */
     private async import(file: TFile): Promise<void> {
-        return this.importer.reload<Partial<PageMetadata>>(file).then(r => {
+        return this.importer.reload<CanvasMetadata | Partial<PageMetadata>>(file).then(r => {
             this.finish(file, r);
             this.persister.storeFile(file.path, r);
         });
     }
 
     /** Finish the reloading of file metadata by adding it to in memory indexes. */
-    private finish(file: TFile, parsed: Partial<PageMetadata>) {
-        let meta = PageMetadata.canonicalize(parsed, link => {
-            let realPath = this.metadataCache.getFirstLinkpathDest(link.path, file.path);
-            if (realPath) return link.withPath(realPath.path);
-            else return link;
-        });
-
-        this.pages.set(file.path, meta);
-        this.tags.set(file.path, meta.fullTags());
-        this.etags.set(file.path, meta.tags);
-        this.links.set(file.path, new Set<string>(meta.links.map(l => l.path)));
+    private finish(file: TFile, parsed: CanvasMetadata | Partial<PageMetadata>) {
+        let meta;
+        if ((parsed as CanvasMetadata).cards) {
+            meta = new CanvasMetadata(
+                file.path,
+                (parsed as CanvasMetadata).cards.map(a => new CanvasCard(a, a.path, file.stat, a)),
+                file.stat,
+                parsed
+            );
+            this.tags.set(file.path, new Set([...meta].map(a => Array.from(a.fullTags())).flat()));
+            this.etags.set(file.path, new Set([...meta].map(a => Array.from(a.tags)).flat()));
+            this.pages.set(file.path, meta);
+            this.links.set(file.path, new Set<string>([...meta].map(l => l.path)));
+        } else {
+            meta = PageMetadata.canonicalize(parsed as Partial<PageMetadata>, link => {
+                let realPath = this.metadataCache.getFirstLinkpathDest(link.path, file.path);
+                if (realPath) return link.withPath(realPath.path);
+                else return link;
+            });
+            this.pages.set(file.path, meta);
+            this.tags.set(file.path, meta.fullTags());
+            this.etags.set(file.path, meta.tags);
+            this.links.set(file.path, new Set<string>(meta.links.map(l => l.path)));
+        }
 
         this.touch();
         this.trigger("update", file);
@@ -297,6 +321,14 @@ export namespace PathFilters {
         let lcPath = path.toLowerCase();
         return lcPath.endsWith(".md") || lcPath.endsWith(".markdown");
     }
+    export function canvas(path: string): boolean {
+        let lcPath = path.toLowerCase();
+        return lcPath.endsWith(".canvas");
+    }
+    export function markdownOrCanvas(path: string): boolean {
+        let lcPath = path.toLowerCase();
+        return lcPath.endsWith(".canvas") || lcPath.endsWith(".md") || lcPath.endsWith(".markdown");
+    }
 }
 
 /**