[wip] add settings-based code extractors

packages/jupyterlab-lsp/package.json

@@ -34,7 +34,7 @@
     "build:labextension:dev": "jupyter labextension build --development True .",
     "build:lib": "tsc",
     "build:prod": "jlpm run build:lib && jlpm run build:labextension",
-    "build:schema": "jlpm build:schema-backend && jlpm build:schema-completion && jlpm build:schema-hover && jlpm build:schema-diagnostics && jlpm build:schema-syntax_highlighting && jlpm build:schema-jump_to && jlpm build:schema-signature && jlpm build:schema-highlights && jlpm build:schema-plugin && jlpm build:schema-rename && jlpm build:schema-symbol",
+    "build:schema": "jlpm build:schema-backend && jlpm build:schema-completion && jlpm build:schema-hover && jlpm build:schema-diagnostics && jlpm build:schema-syntax_highlighting && jlpm build:schema-jump_to && jlpm build:schema-signature && jlpm build:schema-highlights && jlpm build:schema-plugin && jlpm build:schema-rename && jlpm build:schema-symbol && jlpm build:schema-transclusions",
     "build:schema-backend": "json2ts ../../python_packages/jupyter_lsp/jupyter_lsp/schema/schema.json --unreachableDefinitions | prettier --stdin-filepath _schema.d.ts > src/_schema.ts",
     "build:schema-plugin": "json2ts schema/plugin.json | prettier --stdin-filepath _plugin.d.ts > src/_plugin.ts",
     "build:schema-completion": "json2ts schema/completion.json | prettier --stdin-filepath _completion.d.ts > src/_completion.ts ",
@@ -46,6 +46,7 @@
     "build:schema-rename": "json2ts schema/rename.json | prettier --stdin-filepath _rename.d.ts > src/_rename.ts",
     "build:schema-signature": "json2ts schema/signature.json | prettier --stdin-filepath _signature.d.ts > src/_signature.ts",
     "build:schema-symbol": "json2ts schema/symbol.json | prettier --stdin-filepath _symbol.d.ts > src/_symbol.ts",
+    "build:schema-transclusions": "json2ts schema/transclusions.json | prettier --stdin-filepath _transclusions.d.ts > src/transclusions/settings/_transclusions.ts",
     "bundle": "npm pack .",
     "clean": "jlpm run clean:lib",
     "clean:all": "jlpm run clean:lib && jlpm run clean:labextension",

packages/jupyterlab-lsp/schema/transclusions.json

@@ -0,0 +1,101 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "jupyter.lab.setting-icon": "lsp:transclusions",
+  "jupyter.lab.setting-icon-label": "Language integration",
+  "title": "Code Extractors",
+  "description": "Simple declarative patterns for describing \"foreign languages\" embedded within a host language document",
+  "type": "object",
+  "properties": {
+    "enabled": {
+      "title": "Enable Custom Code Extractors",
+      "type": "boolean",
+      "description": "Whether settings-based extractors will be used to find embedded 'foreign' language fragments within a 'host' language.",
+      "default": true
+    },
+    "codeExtractors": {
+      "title": "Code Extractors",
+      "description": "A named set of patterns that find language fragments embedded in other languages",
+      "additionalProperties": {
+        "$ref": "#/definitions/a-code-extractor"
+      }
+    }
+  },
+  "definitions": {
+    "a-language-name": {
+      "anyOf": [
+        {
+          "title": "Well-known language",
+          "$ref": "#/definitions/a-well-known-language"
+        },
+        {
+          "title": "Custom Language",
+          "type": "string"
+        }
+      ]
+    },
+    "a-well-known-language": {
+      "type": "string",
+      "enum": ["javascript", "ipythongfm"]
+    },
+    "a-code-extractor": {
+      "type": "object",
+      "required": [
+        "pattern",
+        "fileExtension",
+        "isStandalone",
+        "hostLanguage",
+        "foreignLanguage",
+        "foreignCaptureGroups",
+        "cellTypes"
+      ],
+      "properties": {
+        "pattern": {
+          "title": "Pattern",
+          "type": "string",
+          "description": "Regular expression to test cells for the foreign language presence."
+        },
+        "hostLanguage": {
+          "title": "Host Language",
+          "description": "Name of the language in which to look for foreign language fragments.",
+          "type": "string",
+          "$ref": "#/definitions/a-language-name"
+        },
+        "foreignLanguage": {
+          "title": "Foreign Language",
+          "description": "Name of the language for an embedded fragment",
+          "type": "string",
+          "$ref": "#/definitions/a-language-name"
+        },
+        "foreignCaptureGroups": {
+          "type": "array",
+          "description": "Array of numbers specifying match groups to be extracted from the regular expression match, for the use in virtual document of the foreign language",
+          "items": {
+            "type": "number",
+            "minimum": 0
+          },
+          "minItems": 1
+        },
+        "isStandalone": {
+          "type": "boolean",
+          "description": "Should the foreign code be appended (False) to the previously established virtual document of the same language, or is it standalone snippet which requires separate connection?",
+          "default": false
+        },
+        "cellTypes": {
+          "type": "array",
+          "description": "Types of Notebook cells in which to look for guest language fragments",
+          "uniqueItems": true,
+          "minItems": 1,
+          "items": {
+            "type": "string",
+            "enum": ["code", "markdown", "raw"]
+          }
+        },
+        "fileExtension": {
+          "type": "string",
+          "description": "The extension (without a leading period)",
+          "pattern": "^[^\\.].+"
+        }
+      }
+    }
+  }
+}

packages/jupyterlab-lsp/src/extractors/regexp.ts

@@ -67,6 +67,7 @@ export class RegExpForeignCodeExtractor implements IForeignCodeExtractor {
     this.expression = new RegExp(options.pattern);
     this.standalone = this.options.isStandalone;
     this.fileExtension = this.options.fileExtension;
+    this.cellType = this.options.cellTypes || ['code'];
   }
 
   hasForeignCode(code: string): boolean {
@@ -186,7 +187,7 @@ export class RegExpForeignCodeExtractor implements IForeignCodeExtractor {
   }
 }
 
-namespace RegExpForeignCodeExtractor {
+export namespace RegExpForeignCodeExtractor {
   export interface IOptions {
     /**
      * The foreign language.
@@ -249,6 +250,13 @@ namespace RegExpForeignCodeExtractor {
      * or is it standalone snippet which requires separate connection?
      */
     isStandalone: boolean;
+    /**
+     * The file extension (without a leading `.`) to create for extracted virtual documents
+     */
     fileExtension: string;
+    /**
+     * Cell types in which this extractor should be checked for presence.
+     */
+    cellTypes?: ('code' | 'markdown' | 'raw')[];
   }
 }

packages/jupyterlab-lsp/src/index.ts

@@ -54,6 +54,7 @@ import {
   TLanguageServerConfigurations
 } from './tokens';
 import { DEFAULT_TRANSCLUSIONS } from './transclusions/defaults';
+import { SETTINGS_TRANSCLUSIONS } from './transclusions/settings';
 import { LOG_CONSOLE } from './virtual/console';
 
 const PLUGIN_ID = PLUGIN_ID_BASE + ':plugin';
@@ -258,6 +259,7 @@ const plugins: JupyterFrontEndPlugin<any>[] = [
   NOTEBOOK_ADAPTER_PLUGIN,
   FILEEDITOR_ADAPTER_PLUGIN,
   plugin,
+  SETTINGS_TRANSCLUSIONS,
   ...DEFAULT_TRANSCLUSIONS,
   ...DEFAULT_FEATURES,
   COMPLETION_FALLBACK_PLUGIN

packages/jupyterlab-lsp/src/transclusions/settings/index.ts

@@ -0,0 +1,26 @@
+import {
+  JupyterFrontEnd,
+  JupyterFrontEndPlugin
+} from '@jupyterlab/application';
+import { ILSPCodeExtractorsManager } from '@jupyterlab/lsp';
+import { ISettingRegistry } from '@jupyterlab/settingregistry';
+
+import { CustomTransclusionsManager } from './manager';
+import { PLUGIN_ID, ILSPCustomTransclusionsManager } from './tokens';
+
+export const SETTINGS_TRANSCLUSIONS: JupyterFrontEndPlugin<ILSPCustomTransclusionsManager> =
+  {
+    id: PLUGIN_ID,
+    autoStart: true,
+    requires: [ISettingRegistry, ILSPCodeExtractorsManager],
+    activate: (
+      app: JupyterFrontEnd,
+      settingsRegistry: ISettingRegistry,
+      extractorsManager: ILSPCodeExtractorsManager
+    ): ILSPCustomTransclusionsManager => {
+      const options = { settingsRegistry, extractorsManager };
+      const manager = new CustomTransclusionsManager(options);
+      void manager.initialize().catch(console.warn);
+      return manager;
+    }
+  };