From ac0ee2f437f58109f1dbe79e21a12ea9248f86be Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 09:58:39 +0100 Subject: [PATCH 01/15] dev: Support updating TextDocuments --- packages/cspell-lib/api/api.d.ts | 11 +++- .../src/Models/TextDocument.test.ts | 30 ++++++++++- .../cspell-lib/src/Models/TextDocument.ts | 53 ++++++++++++++++++- .../src/__snapshots__/index.test.ts.snap | 1 + packages/cspell-lib/src/index.ts | 2 +- 5 files changed, 92 insertions(+), 5 deletions(-) diff --git a/packages/cspell-lib/api/api.d.ts b/packages/cspell-lib/api/api.d.ts index 83d831bfdf6..bcbc2bd9bdf 100644 --- a/packages/cspell-lib/api/api.d.ts +++ b/packages/cspell-lib/api/api.d.ts @@ -376,6 +376,10 @@ interface Position { line: number; character: number; } +/** + * Range offset tuple. + */ +declare type SimpleRange$1 = [start: number, end: number]; interface TextDocumentLine { readonly text: string; readonly offset: number; @@ -420,7 +424,12 @@ interface CreateTextDocumentParams { locale?: string | undefined; version?: number | undefined; } +interface TextDocumentContentChangeEvent { + range?: SimpleRange$1; + text: string; +} declare function createTextDocument({ uri, content, languageId, locale, version, }: CreateTextDocumentParams): TextDocument; +declare function updateTextDocument(doc: TextDocument, edits: TextDocumentContentChangeEvent[], version?: number): TextDocument; /** * Handles loading of `.pnp.js` and `.pnp.js` files. @@ -783,4 +792,4 @@ declare function resolveFile(filename: string, relativeTo: string): ResolveFileR declare function clearCachedFiles(): Promise; declare function getDictionary(settings: CSpellUserSettings): Promise; -export { CheckTextInfo, ConfigurationDependencies, CreateTextDocumentParams, DetermineFinalDocumentSettingsResult, Document, DocumentValidator, DocumentValidatorOptions, ENV_CSPELL_GLOB_ROOT, ExcludeFilesGlobMap, ExclusionFunction, exclusionHelper_d as ExclusionHelper, ImportError, ImportFileRefWithError, IncludeExcludeFlag, IncludeExcludeOptions, index_link_d as Link, Logger, SpellCheckFileOptions, SpellCheckFileResult, SpellingDictionary, SpellingDictionaryCollection, SpellingDictionaryLoadError, SuggestOptions, SuggestedWord, SuggestionError, SuggestionOptions, SuggestionsForWordResult, text_d as Text, TextDocument, TextDocumentLine, TextInfoItem, TraceOptions, TraceResult, ValidationIssue, calcOverrideSettings, checkFilenameMatchesGlob, checkText, clearCachedFiles, clearCachedSettingsFiles, combineTextAndLanguageSettings, combineTextAndLanguageSettings as constructSettingsForText, createSpellingDictionary, createTextDocument, currentSettingsFileVersion, defaultConfigFilenames, defaultFileName, defaultFileName as defaultSettingsFilename, determineFinalDocumentSettings, extractDependencies, extractImportErrors, fileToDocument, finalizeSettings, getCachedFileSize, getDefaultSettings, getDictionary, getGlobalSettings, getLanguagesForExt, getLogger, getSources, isBinaryFile, isSpellingDictionaryLoadError, loadConfig, loadPnP, loadPnPSync, mergeInDocSettings, mergeSettings, readRawSettings, readSettings, readSettingsFiles, refreshDictionaryCache, resolveFile, searchForConfig, sectionCSpell, setLogger, spellCheckDocument, spellCheckFile, suggestionsForWord, suggestionsForWords, traceWords, traceWordsAsync, validateText }; +export { CheckTextInfo, ConfigurationDependencies, CreateTextDocumentParams, DetermineFinalDocumentSettingsResult, Document, DocumentValidator, DocumentValidatorOptions, ENV_CSPELL_GLOB_ROOT, ExcludeFilesGlobMap, ExclusionFunction, exclusionHelper_d as ExclusionHelper, ImportError, ImportFileRefWithError, IncludeExcludeFlag, IncludeExcludeOptions, index_link_d as Link, Logger, SpellCheckFileOptions, SpellCheckFileResult, SpellingDictionary, SpellingDictionaryCollection, SpellingDictionaryLoadError, SuggestOptions, SuggestedWord, SuggestionError, SuggestionOptions, SuggestionsForWordResult, text_d as Text, TextDocument, TextDocumentLine, TextInfoItem, TraceOptions, TraceResult, ValidationIssue, calcOverrideSettings, checkFilenameMatchesGlob, checkText, clearCachedFiles, clearCachedSettingsFiles, combineTextAndLanguageSettings, combineTextAndLanguageSettings as constructSettingsForText, createSpellingDictionary, createTextDocument, currentSettingsFileVersion, defaultConfigFilenames, defaultFileName, defaultFileName as defaultSettingsFilename, determineFinalDocumentSettings, extractDependencies, extractImportErrors, fileToDocument, finalizeSettings, getCachedFileSize, getDefaultSettings, getDictionary, getGlobalSettings, getLanguagesForExt, getLogger, getSources, isBinaryFile, isSpellingDictionaryLoadError, loadConfig, loadPnP, loadPnPSync, mergeInDocSettings, mergeSettings, readRawSettings, readSettings, readSettingsFiles, refreshDictionaryCache, resolveFile, searchForConfig, sectionCSpell, setLogger, spellCheckDocument, spellCheckFile, suggestionsForWord, suggestionsForWords, traceWords, traceWordsAsync, updateTextDocument, validateText }; diff --git a/packages/cspell-lib/src/Models/TextDocument.test.ts b/packages/cspell-lib/src/Models/TextDocument.test.ts index dba242009c9..5756762604a 100644 --- a/packages/cspell-lib/src/Models/TextDocument.test.ts +++ b/packages/cspell-lib/src/Models/TextDocument.test.ts @@ -1,4 +1,4 @@ -import { createTextDocument } from './TextDocument'; +import { createTextDocument, updateTextDocument } from './TextDocument'; import { Uri } from './Uri'; describe('TextDocument', () => { @@ -10,4 +10,32 @@ describe('TextDocument', () => { expect(doc.uri).toBeInstanceOf(Uri); expect(doc.uri.toString()).toEqual(Uri.file(__filename).toString()); }); + + test('update', () => { + const doc = sampleDoc(); + expect(doc.version).toBe(1); + const t = 'self'; + const textCopy = doc.text; + const offset = doc.text.indexOf(t); + updateTextDocument(doc, [{ range: [offset, offset + t.length], text: 'showSelf' }]); + expect(doc.version).toBe(2); + expect(doc.text).not.toEqual(textCopy); + expect(doc.text.startsWith('showSelf', offset)).toBe(true); + updateTextDocument(doc, [{ text: textCopy }]); + expect(doc.text).toBe(textCopy); + }); }); + +function sampleDoc(filename?: string, content?: string) { + filename = filename ?? __filename; + content = + content ?? + ` +import * as fs from 'fs'; + +export function self(): string { + return fs.readFileSync(__filename, 'utf8'); +} +`; + return createTextDocument({ uri: filename, content }); +} diff --git a/packages/cspell-lib/src/Models/TextDocument.ts b/packages/cspell-lib/src/Models/TextDocument.ts index 23f570abdbb..3c4b368768d 100644 --- a/packages/cspell-lib/src/Models/TextDocument.ts +++ b/packages/cspell-lib/src/Models/TextDocument.ts @@ -1,6 +1,7 @@ import { getLanguagesForBasename } from '../LanguageIds'; import * as Uri from './Uri'; import { TextDocument as VsTextDocument } from 'vscode-languageserver-textdocument'; +import assert from 'assert'; export type DocumentUri = Uri.Uri; @@ -9,6 +10,11 @@ export interface Position { character: number; } +/** + * Range offset tuple. + */ +export type SimpleRange = [start: number, end: number]; + export interface TextDocumentLine { readonly text: string; readonly offset: number; @@ -55,15 +61,23 @@ class TextDocumentImpl implements TextDocument { constructor( readonly uri: DocumentUri, - readonly text: string, + text: string, readonly languageId: string | string[], readonly locale: string | undefined, - readonly version: number + version: number ) { const primaryLanguageId = typeof languageId === 'string' ? languageId : languageId[0] || 'plaintext'; this.vsTextDoc = VsTextDocument.create(uri.toString(), primaryLanguageId, version, text); } + get version(): number { + return this.vsTextDoc.version; + } + + get text(): string { + return this.vsTextDoc.getText(); + } + positionAt(offset: number): Position { return this.vsTextDoc.positionAt(offset); } @@ -90,6 +104,27 @@ class TextDocumentImpl implements TextDocument { position, }; } + + /** + * Apply edits to the text. + * Note: the edits are applied one after the other. + * @param edits - changes to the text + * @param version - optional version to use. + * @returns this + */ + update(edits: TextDocumentContentChangeEvent[], version?: number): this { + version = version ?? this.version + 1; + for (const edit of edits) { + const vsEdit = edit.range + ? { + range: { start: this.positionAt(edit.range[0]), end: this.positionAt(edit.range[1]) }, + text: edit.text, + } + : edit; + VsTextDocument.update(this.vsTextDoc, [vsEdit], version); + } + return this; + } } export interface CreateTextDocumentParams { @@ -100,6 +135,11 @@ export interface CreateTextDocumentParams { version?: number | undefined; } +export interface TextDocumentContentChangeEvent { + range?: SimpleRange; + text: string; +} + export function createTextDocument({ uri, content, @@ -113,3 +153,12 @@ export function createTextDocument({ languageId = languageId.length === 0 ? 'text' : languageId; return new TextDocumentImpl(uri, content, languageId, locale, version); } + +export function updateTextDocument( + doc: TextDocument, + edits: TextDocumentContentChangeEvent[], + version?: number +): TextDocument { + assert(doc instanceof TextDocumentImpl, 'Unknown TextDocument type'); + return doc.update(edits, version); +} diff --git a/packages/cspell-lib/src/__snapshots__/index.test.ts.snap b/packages/cspell-lib/src/__snapshots__/index.test.ts.snap index 76e61965e20..798dffdcd1d 100644 --- a/packages/cspell-lib/src/__snapshots__/index.test.ts.snap +++ b/packages/cspell-lib/src/__snapshots__/index.test.ts.snap @@ -182,6 +182,7 @@ Object { "suggestionsForWords": [Function], "traceWords": [Function], "traceWordsAsync": [Function], + "updateTextDocument": [Function], "validateText": [Function], "writeToFile": [Function], "writeToFileIterable": [Function], diff --git a/packages/cspell-lib/src/index.ts b/packages/cspell-lib/src/index.ts index d1ae66f762d..d565785c50e 100644 --- a/packages/cspell-lib/src/index.ts +++ b/packages/cspell-lib/src/index.ts @@ -10,7 +10,7 @@ export * from '@cspell/cspell-types'; export * from 'cspell-io'; export { ExcludeFilesGlobMap, ExclusionFunction } from './exclusionHelper'; export { getLanguagesForExt } from './LanguageIds'; -export { createTextDocument } from './Models/TextDocument'; +export { createTextDocument, updateTextDocument } from './Models/TextDocument'; export type { CreateTextDocumentParams, TextDocument, TextDocumentLine } from './Models/TextDocument'; export * from './Settings'; export { defaultFileName as defaultSettingsFilename } from './Settings'; From 6ff8b671c16558554804aedc3404ffa4e07dcc18 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 10:00:06 +0100 Subject: [PATCH 02/15] refactor: remove unnecessary steps when loading dicts --- .../src/Settings/DictionarySettings.test.ts | 9 +++++---- .../cspell-lib/src/Settings/DictionarySettings.ts | 13 +++++-------- .../src/SpellingDictionary/Dictionaries.test.ts | 7 +++++-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/packages/cspell-lib/src/Settings/DictionarySettings.test.ts b/packages/cspell-lib/src/Settings/DictionarySettings.test.ts index c09e18bb6ca..ffd556ffa3e 100644 --- a/packages/cspell-lib/src/Settings/DictionarySettings.test.ts +++ b/packages/cspell-lib/src/Settings/DictionarySettings.test.ts @@ -4,6 +4,7 @@ import * as fsp from 'fs-extra'; import * as os from 'os'; import * as path from 'path'; import { getDefaultSettings } from './DefaultSettings'; +import { createDictionaryReferenceCollection as createRefCol } from './DictionaryReferenceCollection'; import * as DictSettings from './DictionarySettings'; const defaultSettings = getDefaultSettings(); @@ -12,7 +13,7 @@ const oc = expect.objectContaining; describe('Validate DictionarySettings', () => { test('expects default to not be empty', () => { const mapDefs = DictSettings.filterDictDefsToLoad( - ['php', 'wordsEn', 'unknown', 'en_us'], + createRefCol(['php', 'wordsEn', 'unknown', 'en_us']), defaultSettings.dictionaryDefinitions! ); const files = mapDefs.map((def) => def.name!); @@ -34,7 +35,7 @@ describe('Validate DictionarySettings', () => { 'en_us', ]; const expected = ['php', 'en_us'].sort(); - const mapDefs = DictSettings.filterDictDefsToLoad(ids, defaultSettings.dictionaryDefinitions!); + const mapDefs = DictSettings.filterDictDefsToLoad(createRefCol(ids), defaultSettings.dictionaryDefinitions!); const dicts = mapDefs.map((def) => def.name!).sort(); expect(dicts).toEqual(expected); }); @@ -45,7 +46,7 @@ describe('Validate DictionarySettings', () => { ${'!php, php, !!php, !!cpp'} | ${'cpp, php'} ${'!!!!!!!!!!cpp, !cpp'} | ${'cpp'} `('validate dictionary exclusions $ids', ({ ids, expected }: { ids: string; expected: string }) => { - const dictIds = ids.split(','); + const dictIds = createRefCol(ids.split(',')); const expectedIds = expected.split(',').map((id) => id.trim()); const mapDefs = DictSettings.filterDictDefsToLoad(dictIds, defaultSettings.dictionaryDefinitions!); const dicts = mapDefs.map((def) => def.name!).sort(); @@ -54,7 +55,7 @@ describe('Validate DictionarySettings', () => { test('tests that the files exist', () => { const defaultDicts = defaultSettings.dictionaryDefinitions!; - const dictIds = defaultDicts.map((def) => def.name); + const dictIds = createRefCol(defaultDicts.map((def) => def.name)); const mapDefs = DictSettings.filterDictDefsToLoad(dictIds, defaultSettings.dictionaryDefinitions!); const access = mapDefs.map((def) => def.path!).map((path) => fsp.access(path)); expect(mapDefs.length).toBeGreaterThan(0); diff --git a/packages/cspell-lib/src/Settings/DictionarySettings.ts b/packages/cspell-lib/src/Settings/DictionarySettings.ts index 912791ed0b0..947c7d714c1 100644 --- a/packages/cspell-lib/src/Settings/DictionarySettings.ts +++ b/packages/cspell-lib/src/Settings/DictionarySettings.ts @@ -4,7 +4,6 @@ import type { DictionaryDefinitionAugmented, DictionaryDefinitionCustom, DictionaryFileTypes, - DictionaryReference, ReplaceMap, } from '@cspell/cspell-types'; import * as path from 'path'; @@ -14,7 +13,7 @@ import { DictionaryDefinitionInternal, DictionaryDefinitionInternalWithSource, } from '../Models/CSpellSettingsInternalDef'; -import { createDictionaryReferenceCollection } from './DictionaryReferenceCollection'; +import { createDictionaryReferenceCollection, DictionaryReferenceCollection } from './DictionaryReferenceCollection'; import { mapDictionaryInformationToWeightMap, WeightMap } from 'cspell-trie-lib'; import { DictionaryInformation } from '@cspell/cspell-types'; import { RequireOptional, UnionFields } from '../util/types'; @@ -30,17 +29,15 @@ export type DefMapArrayItem = [string, DictionaryDefinitionInternal]; * - Adding `!` to a dictId will remove the dictionary. * - Adding `!!` will add it back. * - * @param dictIds - dictionaries desired + * @param dictRefCol - dictionaries desired * @param defs - dictionary definitions * @returns map from dictIds to definitions */ export function filterDictDefsToLoad( - dictRefIds: DictionaryReference[], + dictRefCol: DictionaryReferenceCollection, defs: DictionaryDefinitionInternal[] ): DictionaryDefinitionInternal[] { - const col = createDictionaryReferenceCollection(dictRefIds); - const dictIdSet = new Set(col.enabled()); - const allActiveDefs = defs.filter(({ name }) => dictIdSet.has(name)).map(fixPath); + const allActiveDefs = defs.filter(({ name }) => dictRefCol.isEnabled(name)).map(fixPath); return [...new Map(allActiveDefs.map((d) => [d.name, d])).values()]; } @@ -106,7 +103,7 @@ export function calcDictionaryDefsToLoad(settings: CSpellSettingsInternal): Dict if (enabled === undefined) return def; return { ...def, noSuggest: enabled }; }); - return filterDictDefsToLoad(colDicts.enabled(), modDefs); + return filterDictDefsToLoad(colDicts, modDefs); } type DictDef = Partial< diff --git a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.test.ts b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.test.ts index 2a3f8c7fc09..649cd4d5032 100644 --- a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.test.ts +++ b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.test.ts @@ -3,6 +3,7 @@ import * as fs from 'fs-extra'; import * as path from 'path'; import { createCSpellSettingsInternal as csi } from '../Models/CSpellSettingsInternalDef'; import { getDefaultSettings, loadConfig } from '../Settings'; +import { createDictionaryReferenceCollection } from '../Settings/DictionaryReferenceCollection'; import { filterDictDefsToLoad, mapDictDefToInternal } from '../Settings/DictionarySettings'; import * as Dictionaries from './Dictionaries'; import { __testing__ } from './DictionaryLoader'; @@ -191,7 +192,8 @@ describe('Validate Refresh', () => { di({ name: 'not_found', path: tempDictPathNotFound }, __filename), ]); const toLoad = ['node', 'html', 'css', 'not_found', 'temp']; - const defsToLoad = filterDictDefsToLoad(toLoad, defs); + const col = createDictionaryReferenceCollection(toLoad); + const defsToLoad = filterDictDefsToLoad(col, defs); expect(defsToLoad.map((d) => d.name)).toEqual(['css', 'html', 'node', 'temp', 'not_found']); const dicts = await Promise.all(Dictionaries.loadDictionaryDefs(defsToLoad)); @@ -237,7 +239,8 @@ describe('Validate Refresh', () => { di({ name: 'temp', path: tempDictPath }, __filename), ]); const toLoad = ['node', 'html', 'css', 'temp']; - const defsToLoad = filterDictDefsToLoad(toLoad, defs); + const col = createDictionaryReferenceCollection(toLoad); + const defsToLoad = filterDictDefsToLoad(col, defs); expect(defsToLoad.map((d) => d.name)).toEqual(['css', 'html', 'node', 'temp']); const dicts = Dictionaries.loadDictionaryDefsSync(defsToLoad); From 69c88d3fdf50b357e2cd303242d06a042f7d57a7 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 13:27:46 +0100 Subject: [PATCH 03/15] refactor: clean up memorizer signature --- .../src/lib/utils/memorizer.ts | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/cspell-trie-lib/src/lib/utils/memorizer.ts b/packages/cspell-trie-lib/src/lib/utils/memorizer.ts index 7de97667a9f..7eea95e67b6 100644 --- a/packages/cspell-trie-lib/src/lib/utils/memorizer.ts +++ b/packages/cspell-trie-lib/src/lib/utils/memorizer.ts @@ -1,13 +1,9 @@ -export function memorizer(fn: (...p: [...K[]]) => T): (...p: [...K[]]) => T; -export function memorizer(fn: (...p: [K0, K1, K2, K3]) => T): (...p: [K0, K1, K2, K3]) => T; -export function memorizer(fn: (...p: [K0, K1, K2]) => T): (...p: [K0, K1, K2]) => T; -export function memorizer(fn: (...p: [K0, K1]) => T): (...p: [K0, K1]) => T; -export function memorizer(fn: (...p: [K0]) => T): (...p: [K0]) => T; -export function memorizer(fn: (...p: [...K[]]) => T): (...p: [...K[]]) => T { - type N = M; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function memorizer(fn: (...p: K) => T): (...p: K) => T { + type N = M; const r: N = {}; - function find(p: [...K[]]): N | undefined { + function find(p: K): N | undefined { let n: N | undefined = r; for (const k of p) { if (!n) break; @@ -16,7 +12,7 @@ export function memorizer(fn: (...p: [...K[]]) => T): (...p: [...K[]]) => return n; } - function set(p: [...K[]], v: T) { + function set(p: K, v: T) { let n = r; for (const k of p) { const c = n.c?.get(k); @@ -32,7 +28,7 @@ export function memorizer(fn: (...p: [...K[]]) => T): (...p: [...K[]]) => n.v = v; } - return (...p: [...K[]]): T => { + return (...p: K): T => { const f = find(p); if (f && 'v' in f) { return f.v; @@ -43,7 +39,7 @@ export function memorizer(fn: (...p: [...K[]]) => T): (...p: [...K[]]) => }; } -interface M { - v?: T; - c?: Map>; +interface M { + v?: Value; + c?: Map>; } From 84972e208f8835460c6ea85acf1923c95e6b240d Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 13:27:54 +0100 Subject: [PATCH 04/15] Update jest.config.js --- jest.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jest.config.js b/jest.config.js index 877733db4ee..42061f142cb 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,7 +3,7 @@ module.exports = { transform: { '^.+\\.tsx?$': 'ts-jest', }, - testRegex: '(/__tests__/.*|(\\.|/)(test|spec|perf))\\.[jt]sx?$', + testRegex: '(/__tests__/.*|\\.(test|spec|perf))\\.[jt]sx?$', moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], coverageReporters: ['html', 'json', ['lcov', { projectRoot: __dirname }], 'text'], // "coverageProvider": "v8", From 6ffecb21f5761ee6eae940eff5c61db6f464bbbe Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 15:59:55 +0100 Subject: [PATCH 05/15] refactor simpleCache to use boxed values. --- packages/cspell-lib/src/util/simpleCache.ts | 52 ++++++++++++--------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/packages/cspell-lib/src/util/simpleCache.ts b/packages/cspell-lib/src/util/simpleCache.ts index 50fa64ef8f6..72d6ec90541 100644 --- a/packages/cspell-lib/src/util/simpleCache.ts +++ b/packages/cspell-lib/src/util/simpleCache.ts @@ -1,7 +1,8 @@ +type Box = { v: T }; export class SimpleWeakCache { - private L0 = new WeakMap(); - private L1 = new WeakMap(); - private L2 = new WeakMap(); + private L0 = new WeakMap>(); + private L1 = new WeakMap>(); + private L2 = new WeakMap>(); private sizeL0 = 0; constructor(readonly size: number) {} @@ -15,21 +16,24 @@ export class SimpleWeakCache { get(key: K): T | undefined { for (const c of this.caches()) { - if (c.has(key)) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const v = c.get(key)!; + const entry = c.get(key); + if (entry) { if (c !== this.L0) { - this.set(key, v); + this._set(key, entry); } - return v; + return entry.v; } } return undefined; } set(key: K, value: T) { + this._set(key, { v: value }); + } + + private _set(key: K, entry: Box) { if (this.L0.has(key)) { - this.L0.set(key, value); + this.L0.set(key, entry); return this; } @@ -38,7 +42,7 @@ export class SimpleWeakCache { } this.sizeL0 += 1; - this.L0.set(key, value); + this.L0.set(key, entry); } private caches() { @@ -48,7 +52,7 @@ export class SimpleWeakCache { private rotate() { this.L2 = this.L1; this.L1 = this.L0; - this.L0 = new WeakMap(); + this.L0 = new WeakMap>(); this.sizeL0 = 0; } } @@ -77,9 +81,9 @@ export class AutoWeakCache extends SimpleWeakCache { * promoted to L0. */ export class SimpleCache { - private L0 = new Map(); - private L1 = new Map(); - private L2 = new Map(); + private L0 = new Map>(); + private L1 = new Map>(); + private L2 = new Map>(); constructor(readonly size: number) {} @@ -92,21 +96,24 @@ export class SimpleCache { get(key: K): T | undefined { for (const c of this.caches()) { - if (c.has(key)) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const v = c.get(key)!; + const entry = c.get(key); + if (entry) { if (c !== this.L0) { - this.set(key, v); + this._set(key, entry); } - return v; + return entry.v; } } return undefined; } set(key: K, value: T) { + this._set(key, { v: value }); + } + + private _set(key: K, entry: Box) { if (this.L0.has(key)) { - this.L0.set(key, value); + this.L0.set(key, entry); return this; } @@ -114,7 +121,7 @@ export class SimpleCache { this.rotate(); } - this.L0.set(key, value); + this.L0.set(key, entry); } private caches() { @@ -122,10 +129,9 @@ export class SimpleCache { } private rotate() { - this.L2.clear(); this.L2 = this.L1; this.L1 = this.L0; - this.L0 = new Map(); + this.L0 = new Map>(); } } From c97ecf14e800824c9e82c2a5e087f950b88f33b0 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 16:00:56 +0100 Subject: [PATCH 06/15] add intersection test --- packages/cspell-lib/src/util/util.test.ts | 10 ++++++++++ packages/cspell-lib/src/util/util.ts | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/packages/cspell-lib/src/util/util.test.ts b/packages/cspell-lib/src/util/util.test.ts index 2c667f989c7..81e8b83651e 100644 --- a/packages/cspell-lib/src/util/util.test.ts +++ b/packages/cspell-lib/src/util/util.test.ts @@ -61,4 +61,14 @@ describe('Validate util', () => { `('isArrayEqual $a $b', ({ a, b, expected }) => { expect(util.isArrayEqual(a, b)).toBe(expected); }); + + test.each` + a | b | expected + ${[]} | ${[]} | ${false} + ${[1, 2]} | ${['1']} | ${false} + ${['1', 2, 4]} | ${['1']} | ${true} + ${['1']} | ${['1', 2, 4]} | ${true} + `('doSetsIntersect $a vs $b', ({ a, b, expected }) => { + expect(util.doSetsIntersect(new Set(a), new Set(b))).toBe(expected); + }); }); diff --git a/packages/cspell-lib/src/util/util.ts b/packages/cspell-lib/src/util/util.ts index 75151247d23..4a24d78a61c 100644 --- a/packages/cspell-lib/src/util/util.ts +++ b/packages/cspell-lib/src/util/util.ts @@ -92,3 +92,19 @@ export function isArrayEqual(a: K[], b: K[]): boolean { } return isMatch; } + +/** + * Determine if two sets intersect + * @param a - first Set + * @param b - second Set + * @returns true iff any element of `a` is in `b` + */ +export function doSetsIntersect(a: Set, b: Set): boolean { + function compare(a: Set, b: Set) { + for (const item of a) { + if (b.has(item)) return true; + } + return false; + } + return a.size <= b.size ? compare(a, b) : compare(b, a); +} From 2a9cf1ecf95c708aae9d1e712dd0217c08898482 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 16:01:18 +0100 Subject: [PATCH 07/15] Add code to record elapsed time. --- packages/cspell-lib/src/util/timer.test.ts | 24 +++++++++++- packages/cspell-lib/src/util/timer.ts | 44 +++++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/packages/cspell-lib/src/util/timer.test.ts b/packages/cspell-lib/src/util/timer.test.ts index baab99da674..08539d9b707 100644 --- a/packages/cspell-lib/src/util/timer.test.ts +++ b/packages/cspell-lib/src/util/timer.test.ts @@ -1,4 +1,4 @@ -import { createTimer, polyHrTime } from './timer'; +import { createLapRecorder, createTimer, polyHrTime } from './timer'; import { promisify } from 'util'; const delay = promisify(setTimeout); @@ -21,4 +21,26 @@ describe('timer', () => { expect(Math.abs(b1 - a1)).toBeLessThan(2); }); + + test('lap', async () => { + const t = createTimer(); + const a = t.lap(); + await delay(12); + const b = t.lap(); + await delay(1); + expect(b).toBeLessThan(t.elapsed()); + expect(a).toBeLessThan(b); + }); +}); + +describe('LapRecorder', () => { + test('LapRecorder', () => { + const timer = createLapRecorder(); + timer.lap('a'); + timer.lap('b'); + expect(timer.times.length).toBe(2); + expect(timer.times[0][0]).toBe('a'); + expect(timer.times[0][1]).toBe(timer.times[0][2]); + expect(timer.report()).toHaveLength(2); + }); }); diff --git a/packages/cspell-lib/src/util/timer.ts b/packages/cspell-lib/src/util/timer.ts index 31d99848e16..2aaa9ccb90a 100644 --- a/packages/cspell-lib/src/util/timer.ts +++ b/packages/cspell-lib/src/util/timer.ts @@ -8,17 +8,31 @@ export interface Timer { * timer was created / started. */ elapsed(): number; + /** + * Calculate the amount of time in ms since the + * end of the last lap. + */ + lap(): number; } export function createTimer(hrTimeFn = _hrTime): Timer { let start: HRTime = hrTimeFn(); + let lastLap = 0; + function elapsed() { + return toMilliseconds(hrTimeFn(start)); + } return { start() { start = hrTimeFn(); + lastLap = 0; }, - elapsed() { - return toMilliseconds(hrTimeFn(start)); + elapsed, + lap() { + const now = elapsed(); + const diff = now - lastLap; + lastLap = now; + return diff; }, }; } @@ -38,3 +52,29 @@ export function polyHrTime(time?: HRTime): HRTime { const n = (inSeconds - s) * 1.0e9; return [s, n]; } + +export interface LapRecorder { + times: [name: string, lapTime: number, totalTime: number][]; + lap(name: string): void; + report(): string[]; +} + +export function createLapRecorder(): LapRecorder { + const timer = createTimer(); + const times: [string, number, number][] = []; + let lapTime = 0; + function lap(name: string) { + const now = timer.elapsed(); + const diff = now - lapTime; + times.push([name, diff, now]); + lapTime = diff; + } + function report() { + return times.map(([name, time]) => `${name}: ${time.toFixed(2)}`); + } + return { + times, + lap, + report, + }; +} From 8ab3244ae57bd638ab30a357c4c2cb76f81da13b Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 16:27:55 +0100 Subject: [PATCH 08/15] Performance debug helper fn. --- .../cspell-lib/src/util/debugPerf.test.ts | 10 ++++++++ packages/cspell-lib/src/util/debugPerf.ts | 23 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 packages/cspell-lib/src/util/debugPerf.test.ts create mode 100644 packages/cspell-lib/src/util/debugPerf.ts diff --git a/packages/cspell-lib/src/util/debugPerf.test.ts b/packages/cspell-lib/src/util/debugPerf.test.ts new file mode 100644 index 00000000000..eb0caef66aa --- /dev/null +++ b/packages/cspell-lib/src/util/debugPerf.test.ts @@ -0,0 +1,10 @@ +import { perfFn } from './debugPerf'; + +describe('debugPerf', () => { + test('perfFn', () => { + const mock = jest.fn(); + const fn = perfFn(() => undefined, 'message', mock); + fn(); + expect(mock).toHaveBeenCalledWith(expect.stringContaining('message')); + }); +}); diff --git a/packages/cspell-lib/src/util/debugPerf.ts b/packages/cspell-lib/src/util/debugPerf.ts new file mode 100644 index 00000000000..39348180a06 --- /dev/null +++ b/packages/cspell-lib/src/util/debugPerf.ts @@ -0,0 +1,23 @@ +import { createTimer } from './timer'; + +/** + * Measure and log result. + * @param fn - function to measure. + * @param message - message to log + * @param callback - called when the function has finished. + * @returns a function + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function perfFn

( + fn: (...args: P) => R, + message: string, + callback: (m: string, elapsedMs: number) => void = (message, time) => + console.error(`${message}: ${time.toFixed(2)}ms`) +): (...args: P) => R { + return (...args: P) => { + const timer = createTimer(); + const r = fn(...args); + callback(message, timer.elapsed()); + return r; + }; +} From 6be212fc6958c7cab55ec5478c0e65c7999f9d16 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 16:33:06 +0100 Subject: [PATCH 09/15] Avoid creating unnecessary objects in settings. --- .../src/Settings/CSpellSettingsServer.test.ts | 6 -- .../src/Settings/CSpellSettingsServer.ts | 26 +++--- .../src/Settings/LanguageSettings.ts | 86 +++++++++++-------- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts index db80c84d82b..1fd1643b516 100644 --- a/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts +++ b/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts @@ -27,7 +27,6 @@ describe('Validate CSpellSettingsServer', () => { name: 'Left|Right', id: '|', enabledLanguageIds: [], - languageSettings: [], source: { name: 'Left|Right', sources: [left, right] }, }) ); @@ -42,7 +41,6 @@ describe('Validate CSpellSettingsServer', () => { name: '|enabledName', id: 'left|enabledId', enabledLanguageIds: [], - languageSettings: [], source: { name: 'left|enabledName', sources: [csi(left), csi(enabled)] }, }) ); @@ -59,7 +57,6 @@ describe('Validate CSpellSettingsServer', () => { name: '|', id: [left.id, right.id].join('|'), enabledLanguageIds: [], - languageSettings: [], source: { name: 'left|right', sources: [left, right] }, }) ); @@ -74,7 +71,6 @@ describe('Validate CSpellSettingsServer', () => { name: '|', id: '|', enabledLanguageIds: [], - languageSettings: [], source: { name: 'left|right', sources: [csi(a), csi(b)] }, }) ); @@ -106,7 +102,6 @@ describe('Validate CSpellSettingsServer', () => { name: '|', id: [left.id, right.id].join('|'), enabledLanguageIds: [], - languageSettings: [], files: left.files?.concat(right.files || []), ignorePaths: left.ignorePaths?.concat(right.ignorePaths || []), overrides: left.overrides?.concat(right.overrides || []), @@ -144,7 +139,6 @@ describe('Validate CSpellSettingsServer', () => { id: [left.id, right.id].join('|'), version: right.version, enabledLanguageIds: [], - languageSettings: [], files: left.files?.concat(right.files || []), ignorePaths: right.ignorePaths, overrides: right.overrides, diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts index 55acee1f829..03a1f6d9114 100644 --- a/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts +++ b/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts @@ -3,7 +3,6 @@ import type { CSpellUserSettings, Glob, ImportFileRef, - LanguageSetting, Source, } from '@cspell/cspell-types'; import { GlobMatcher } from 'cspell-glob'; @@ -28,6 +27,10 @@ export const configSettingsFileVersion0_2 = '0.2'; export const currentSettingsFileVersion = configSettingsFileVersion0_2; export const ENV_CSPELL_GLOB_ROOT = 'CSPELL_GLOB_ROOT'; +function _unique(a: T[]): T[] { + return [...new Set(a)]; +} + /** * Merges two lists and removes duplicates. Order is NOT preserved. */ @@ -39,8 +42,9 @@ function mergeListUnique(left: T[] | undefined, right: T[] | undefined): T[] function mergeListUnique(left: T[] | undefined, right: T[] | undefined): T[] | undefined { if (left === undefined) return right; if (right === undefined) return left; - const uniqueItems = new Set([...left, ...right]); - return [...uniqueItems.keys()]; + if (!right.length) return left; + if (!left.length) return right; + return _unique([...left, ...right]); } /** @@ -91,13 +95,6 @@ function mergeObjects(left?: T, right?: T): T | undefined { return { ...left, ...right }; } -function tagLanguageSettings(tag: string, settings: LanguageSetting[] = []): LanguageSetting[] { - return settings.map((s) => ({ - id: tag + '.' + (s.id || s.locale || s.languageId), - ...s, - })); -} - function replaceIfNotEmpty(left: Array = [], right: Array = []) { const filtered = right.filter((a) => !!a); if (filtered.length) { @@ -108,9 +105,9 @@ function replaceIfNotEmpty(left: Array = [], right: Array = []) { export function mergeSettings( left: CSpellSettingsWSTO | CSpellSettingsI, - ...settings: (CSpellSettingsWSTO | CSpellSettingsI)[] + ...settings: (CSpellSettingsWSTO | CSpellSettingsI | undefined)[] ): CSpellSettingsI { - const rawSettings = settings.reduce(merge, toInternalSettings(left)); + const rawSettings = settings.filter((a) => !!a).reduce(merge, toInternalSettings(left)); return util.clean(rawSettings); } @@ -166,10 +163,7 @@ function merge( dictionaryDefinitions: mergeListUnique(_left.dictionaryDefinitions, _right.dictionaryDefinitions), dictionaries: mergeListUnique(_left.dictionaries, _right.dictionaries), noSuggestDictionaries: mergeListUnique(_left.noSuggestDictionaries, _right.noSuggestDictionaries), - languageSettings: mergeList( - tagLanguageSettings(leftId, _left.languageSettings), - tagLanguageSettings(rightId, _right.languageSettings) - ), + languageSettings: mergeList(_left.languageSettings, _right.languageSettings), enabled: _right.enabled !== undefined ? _right.enabled : _left.enabled, files: mergeListUnique(_left.files, _right.files), ignorePaths: versionBasedMergeList(_left.ignorePaths, _right.ignorePaths, version), diff --git a/packages/cspell-lib/src/Settings/LanguageSettings.ts b/packages/cspell-lib/src/Settings/LanguageSettings.ts index 156062f77a3..b0660a76bbe 100644 --- a/packages/cspell-lib/src/Settings/LanguageSettings.ts +++ b/packages/cspell-lib/src/Settings/LanguageSettings.ts @@ -1,4 +1,13 @@ -import type { LanguageSetting, CSpellUserSettings, LocaleId, LanguageId, BaseSetting } from '@cspell/cspell-types'; +import type { + BaseSetting, + CSpellUserSettings, + LanguageId, + LanguageSetting, + LocaleId, + Settings, +} from '@cspell/cspell-types'; +import { memorizerAll } from '../util/Memorizer'; +import { doSetsIntersect } from '../util/util'; import * as SpellSettings from './CSpellSettingsServer'; // LanguageSettings are a collection of LanguageSetting. They are applied in order, matching against the languageId. @@ -13,69 +22,79 @@ export function getDefaultLanguageSettings(): LanguageSettings { return defaultLanguageSettings; } -function localesToList(locales: string | string[]): string[] { - locales = typeof locales !== 'string' ? locales.join(',') : locales; +function localesToList(locales: string): string[] { return stringToList(locales.replace(/\s+/g, ',')); } -function stringToList(sList: string | string[]): string[] { - if (typeof sList !== 'string') { - sList = sList.join(','); - } - sList = sList +function stringToList(sList: string): string[] { + return sList .replace(/[|;]/g, ',') .split(',') .map((s) => s.trim()) .filter((s) => !!s); - return sList; } -export function normalizeLanguageId(langId: LanguageId | LanguageId[]): Set { +const _normalizeLanguageId = memorizerAll(__normalizeLanguageId); +function __normalizeLanguageId(langId: LanguageId): Set { const langIds = stringToList(langId); return new Set(langIds.map((a) => a.toLowerCase())); } -function normalizeLanguageIdToString(langId: LanguageId | LanguageId[]): string { - return [...normalizeLanguageId(langId)].join(','); +export function normalizeLanguageId(langId: LanguageId | LanguageId[]): Set { + return _normalizeLanguageId(typeof langId === 'string' ? langId : langId.join(',')); +} + +const _normalizeLocale = memorizerAll(__normalizeLocale); +function __normalizeLocale(locale: LocaleId): Set { + const locales = localesToList(locale); + return new Set(locales.map((locale) => locale.toLowerCase().replace(/[^a-z]/g, ''))); } export function normalizeLocale(locale: LocaleId | LocaleId[]): Set { - locale = localesToList(locale); - return new Set(locale.map((locale) => locale.toLowerCase().replace(/[^a-z]/g, ''))); + locale = typeof locale === 'string' ? locale : locale.join(','); + return _normalizeLocale(locale); } export function isLocaleInSet(locale: LocaleId | LocaleId[], setOfLocals: Set): boolean { const locales = normalizeLocale(locale); - return [...locales.values()].filter((locale) => setOfLocals.has(locale)).length > 0; + return doSetsIntersect(locales, setOfLocals); } export function calcSettingsForLanguage( languageSettings: LanguageSettings, languageId: LanguageId, - locale: LocaleId | LocaleId[] + locale: LocaleId ): BaseSetting { languageId = languageId.toLowerCase(); const allowedLocals = normalizeLocale(locale); - return defaultLanguageSettings - .concat(languageSettings) + const ls: Settings & { languageId?: LanguageId; locale?: LocaleId } = languageSettings .filter((s) => doesLanguageSettingMatchLanguageId(s, languageId)) .filter((s) => !s.locale || s.locale === '*' || isLocaleInSet(s.locale, allowedLocals)) .map((langSetting) => { - const id = normalizeLanguageIdToString(langSetting.locale || langSetting.languageId || 'language'); - const { languageId: _languageId, locale: _local, ...s } = { id, ...langSetting }; + const { languageId: _languageId, locale: _locale, ...s } = langSetting; return s; }) - .reduce( - (langSetting, setting) => ({ - ...SpellSettings.mergeSettings(langSetting, setting), - languageId, - locale, - }), - {} - ); + .reduce((langSetting, setting) => SpellSettings.mergeSettings(langSetting, setting), {}); + ls.languageId = languageId; + ls.locale = locale; + return ls; } +const cacheDoesLanguageSettingMatchLanguageId: WeakMap> = new WeakMap(); + function doesLanguageSettingMatchLanguageId(s: LanguageSetting, languageId: LanguageId): boolean { + const r = cacheDoesLanguageSettingMatchLanguageId.get(s) ?? new Map(); + const f = r.get(languageId); + if (f !== undefined) { + return f; + } + const v = _doesLanguageSettingMatchLanguageId(s, languageId); + r.set(languageId, v); + cacheDoesLanguageSettingMatchLanguageId.set(s, r); + return v; +} + +function _doesLanguageSettingMatchLanguageId(s: LanguageSetting, languageId: LanguageId): boolean { const languageSettingsLanguageIds = s.languageId; if (!languageSettingsLanguageIds || languageSettingsLanguageIds === '*') return true; const ids = normalizeLanguageId(languageSettingsLanguageIds); @@ -87,16 +106,13 @@ function doesLanguageSettingMatchLanguageId(s: LanguageSetting, languageId: Lang } export function calcUserSettingsForLanguage(settings: CSpellUserSettings, languageId: string): CSpellUserSettings { - const { languageSettings = [], language: locale = defaultLocale } = settings; - const defaults = { - allowCompoundWords: settings.allowCompoundWords, - enabled: settings.enabled, - }; + const { languageSettings = [], language: locale = defaultLocale, allowCompoundWords, enabled } = settings; const langSettings = { - ...defaults, + allowCompoundWords, + enabled, ...calcSettingsForLanguage(languageSettings, languageId, locale), }; - return SpellSettings.mergeSettings(settings, langSettings as CSpellUserSettings); + return SpellSettings.mergeSettings(settings, langSettings); } export function calcSettingsForLanguageId( From ccfb1efc4fa1f150382e049be6a367a0ae486f75 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 16:36:26 +0100 Subject: [PATCH 10/15] cache suggestions --- .../src/textValidation/docValidator.ts | 78 +++++++++++++++---- 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/packages/cspell-lib/src/textValidation/docValidator.ts b/packages/cspell-lib/src/textValidation/docValidator.ts index a22c7456d29..32bab2d877e 100644 --- a/packages/cspell-lib/src/textValidation/docValidator.ts +++ b/packages/cspell-lib/src/textValidation/docValidator.ts @@ -2,12 +2,13 @@ import { opConcatMap, pipeSync } from '@cspell/cspell-pipe'; import type { CSpellSettingsWithSourceTrace, CSpellUserSettings, PnPSettings } from '@cspell/cspell-types'; import assert from 'assert'; import { CSpellSettingsInternal } from '../Models/CSpellSettingsInternalDef'; -import { TextDocument } from '../Models/TextDocument'; +import { TextDocument, updateTextDocument } from '../Models/TextDocument'; import { finalizeSettings, loadConfig, mergeSettings, searchForConfig } from '../Settings'; import { loadConfigSync, searchForConfigSync } from '../Settings/configLoader'; import { getDictionaryInternal, getDictionaryInternalSync, SpellingDictionaryCollection } from '../SpellingDictionary'; import { toError } from '../util/errors'; import { callOnce } from '../util/Memorizer'; +import { AutoCache } from '../util/simpleCache'; import { MatchRange } from '../util/TextRange'; import { createTimer } from '../util/timer'; import { clean } from '../util/util'; @@ -46,6 +47,7 @@ export class DocumentValidator { private _prepared: Promise | undefined; private _preparations: Preparations | undefined; private _preparationTime = -1; + private _suggestions = new AutoCache((text: string) => this.genSuggestions(text), 1000); /** * @param doc - Document to validate @@ -82,7 +84,7 @@ export class DocumentValidator { this.addPossibleError(localConfig?.__importRef?.error); - const config = localConfig ? mergeSettings(settings, localConfig) : settings; + const config = mergeSettings(settings, localConfig); const docSettings = determineTextDocumentSettings(this._document, config); const dict = getDictionaryInternalSync(docSettings); @@ -94,6 +96,7 @@ export class DocumentValidator { const lineValidator = lineValidatorFactory(dict, validateOptions); this._preparations = { + config, dictionary: dict, docSettings, shouldCheck, @@ -105,6 +108,7 @@ export class DocumentValidator { this._ready = true; this._preparationTime = timer.elapsed(); + console.error(`prepareSync ${this._preparationTime.toFixed(2)}ms`); } async prepare(): Promise { @@ -128,11 +132,11 @@ export class DocumentValidator { : useSearchForConfig ? this.catchError(searchForDocumentConfig(this._document, settings, settings)) : undefined; - const localConfig = await pLocalConfig; + const localConfig = (await pLocalConfig) || {}; this.addPossibleError(localConfig?.__importRef?.error); - const config = localConfig ? mergeSettings(settings, localConfig) : settings; + const config = mergeSettings(settings, localConfig); const docSettings = determineTextDocumentSettings(this._document, config); const dict = await getDictionaryInternal(docSettings); @@ -144,6 +148,7 @@ export class DocumentValidator { const lineValidator = lineValidatorFactory(dict, validateOptions); this._preparations = { + config, dictionary: dict, docSettings, shouldCheck, @@ -157,6 +162,32 @@ export class DocumentValidator { this._preparationTime = timer.elapsed(); } + private _updatePrep() { + assert(this._preparations); + const timer = createTimer(); + const { config } = this._preparations; + const docSettings = determineTextDocumentSettings(this._document, config); + const dict = getDictionaryInternalSync(docSettings); + const shouldCheck = docSettings.enabled ?? true; + const finalSettings = finalizeSettings(docSettings); + const validateOptions = settingsToValidateOptions(finalSettings); + const includeRanges = calcTextInclusionRanges(this._document.text, validateOptions); + const segmenter = mapLineSegmentAgainstRangesFactory(includeRanges); + const lineValidator = lineValidatorFactory(dict, validateOptions); + + this._preparations = { + config, + dictionary: dict, + docSettings, + shouldCheck, + validateOptions, + includeRanges, + segmenter, + lineValidator, + }; + this._preparationTime = timer.elapsed(); + } + /** * The amount of time in ms to prepare for validation. */ @@ -188,20 +219,10 @@ export class DocumentValidator { if (!this.options.generateSuggestions) { return issues; } - const settings = this._preparations.docSettings; - const dict = this._preparations.dictionary; - const sugOptions = clean({ - compoundMethod: 0, - numSuggestions: this.options.numSuggestions, - includeTies: false, - ignoreCase: !(settings.caseSensitive ?? false), - timeout: settings.suggestionsTimeout, - numChanges: settings.suggestionNumChanges, - }); const withSugs = issues.map((t) => { // lazy suggestion calculation. const text = t.text; - const suggestions = callOnce(() => dict.suggest(text, sugOptions).map((r) => r.word)); + const suggestions = callOnce(() => this.suggest(text)); return Object.defineProperty({ ...t }, 'suggestions', { enumerable: true, get: suggestions }); }); @@ -212,6 +233,11 @@ export class DocumentValidator { return this._document; } + public updateDocumentText(text: string) { + updateTextDocument(this._document, [{ text }]); + this._updatePrep(); + } + private addPossibleError(error: Error | undefined | unknown) { if (!error) return; error = this.errors.push(toError(error)); @@ -231,6 +257,25 @@ export class DocumentValidator { } return undefined; } + + private suggest(text: string) { + return this._suggestions.get(text); + } + + private genSuggestions(text: string): string[] { + assert(this._preparations); + const settings = this._preparations.docSettings; + const dict = this._preparations.dictionary; + const sugOptions = clean({ + compoundMethod: 0, + numSuggestions: this.options.numSuggestions, + includeTies: false, + ignoreCase: !(settings.caseSensitive ?? false), + timeout: settings.suggestionsTimeout, + numChanges: settings.suggestionNumChanges, + }); + return dict.suggest(text, sugOptions).map((r) => r.word); + } } export type Offset = number; @@ -238,7 +283,10 @@ export type Offset = number; export type SimpleRange = readonly [Offset, Offset]; interface Preparations { + /** loaded config */ + config: CSpellSettingsInternal; dictionary: SpellingDictionaryCollection; + /** configuration after applying in-doc settings */ docSettings: CSpellSettingsInternal; includeRanges: MatchRange[]; lineValidator: LineValidator; From 90884c2eb58ec9d49026a1d0937a54ad919d9c10 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 16:36:39 +0100 Subject: [PATCH 11/15] Update Memorizer.ts --- packages/cspell-lib/src/util/Memorizer.ts | 46 +++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/packages/cspell-lib/src/util/Memorizer.ts b/packages/cspell-lib/src/util/Memorizer.ts index bb7692c1991..0d8988b8c8a 100644 --- a/packages/cspell-lib/src/util/Memorizer.ts +++ b/packages/cspell-lib/src/util/Memorizer.ts @@ -114,3 +114,49 @@ export function callOnce(fn: () => T): () => T { return last.value; }; } + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function memorizerAll(fn: (...p: K) => T): (...p: K) => T { + type N = M; + const r: N = {}; + + function find(p: K): N | undefined { + let n: N | undefined = r; + for (const k of p) { + if (!n) break; + n = n.c?.get(k); + } + return n; + } + + function set(p: K, v: T) { + let n = r; + for (const k of p) { + const c = n.c?.get(k); + if (c) { + n = c; + continue; + } + const r: N = {}; + n.c = n.c || new Map(); + n.c.set(k, r); + n = r; + } + n.v = v; + } + + return (...p: K): T => { + const f = find(p); + if (f && 'v' in f) { + return f.v; + } + const v = fn(...p); + set(p, v); + return v; + }; +} + +interface M { + v?: Value; + c?: Map>; +} From e233b3549cb56a502f1a6ac9db13bd35a49a27b0 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 16:38:27 +0100 Subject: [PATCH 12/15] improve eslint-plugin performance --- .../src/cspell-eslint-plugin.ts | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/packages/cspell-eslint-plugin/src/cspell-eslint-plugin.ts b/packages/cspell-eslint-plugin/src/cspell-eslint-plugin.ts index 08fe58935d0..d9bdc9ee92f 100644 --- a/packages/cspell-eslint-plugin/src/cspell-eslint-plugin.ts +++ b/packages/cspell-eslint-plugin/src/cspell-eslint-plugin.ts @@ -1,7 +1,7 @@ // cspell:ignore TSESTree import type { TSESTree } from '@typescript-eslint/types'; import assert from 'assert'; -import { createTextDocument, CSpellSettings, DocumentValidator, ValidationIssue } from 'cspell-lib'; +import { createTextDocument, CSpellSettings, DocumentValidator, ValidationIssue, TextDocument } from 'cspell-lib'; import type { Rule } from 'eslint'; // eslint-disable-next-line node/no-missing-import import type { Comment, Identifier, Literal, Node, TemplateElement, ImportSpecifier } from 'estree'; @@ -60,8 +60,7 @@ function create(context: Rule.RuleContext): Rule.RuleListener { const importedIdentifiers = new Set(); isDebugMode = options.debugMode || false; isDebugMode && logContext(context); - const doc = createTextDocument({ uri: context.getFilename(), content: context.getSourceCode().getText() }); - const validator = new DocumentValidator(doc, options, defaultSettings); + const validator = getDocValidator(context); validator.prepareSync(); function checkLiteral(node: Literal & Rule.NodeParentExtension) { @@ -369,3 +368,40 @@ export const configs = { }, }, }; + +interface CachedDoc { + filename: string; + doc: TextDocument; +} + +const cache: { lastDoc: CachedDoc | undefined } = { lastDoc: undefined }; + +const docValCache = new WeakMap(); + +function getDocValidator(context: Rule.RuleContext): DocumentValidator { + const text = context.getSourceCode().getText(); + const doc = getTextDocument(context.getFilename(), text); + const cachedValidator = docValCache.get(doc); + if (cachedValidator) { + cachedValidator.updateDocumentText(text); + return cachedValidator; + } + + const options = normalizeOptions(context.options[0]); + isDebugMode = options.debugMode || false; + isDebugMode && logContext(context); + const validator = new DocumentValidator(doc, options, defaultSettings); + docValCache.set(doc, validator); + return validator; +} + +function getTextDocument(filename: string, content: string): TextDocument { + if (cache.lastDoc?.filename === filename) { + return cache.lastDoc.doc; + } + + const doc = createTextDocument({ uri: filename, content }); + // console.error(`CreateTextDocument: ${doc.uri}`); + cache.lastDoc = { filename, doc }; + return doc; +} From 14e9add94164b5e12c4927d214a0a8e806d2d6d6 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 16:45:32 +0100 Subject: [PATCH 13/15] Update debugPerf.test.ts --- packages/cspell-lib/src/util/debugPerf.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/cspell-lib/src/util/debugPerf.test.ts b/packages/cspell-lib/src/util/debugPerf.test.ts index eb0caef66aa..5d4ec90fe4a 100644 --- a/packages/cspell-lib/src/util/debugPerf.test.ts +++ b/packages/cspell-lib/src/util/debugPerf.test.ts @@ -1,10 +1,17 @@ import { perfFn } from './debugPerf'; describe('debugPerf', () => { - test('perfFn', () => { + test('perfFn with callback', () => { const mock = jest.fn(); const fn = perfFn(() => undefined, 'message', mock); fn(); + expect(mock).toHaveBeenCalledWith(expect.stringContaining('message'), expect.any(Number)); + }); + + test('perfFn default callback', () => { + const mock = jest.spyOn(console, 'error').mockImplementation(); + const fn = perfFn(() => undefined, 'message'); + fn(); expect(mock).toHaveBeenCalledWith(expect.stringContaining('message')); }); }); From 0ddc126fa2cf5c052c351d0dceca93e9649346e5 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 20:28:22 +0100 Subject: [PATCH 14/15] Update api.d.ts --- packages/cspell-lib/api/api.d.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/cspell-lib/api/api.d.ts b/packages/cspell-lib/api/api.d.ts index bcbc2bd9bdf..1d0edd67bc6 100644 --- a/packages/cspell-lib/api/api.d.ts +++ b/packages/cspell-lib/api/api.d.ts @@ -478,7 +478,7 @@ declare type CSpellSettingsWSTO = OptionalOrUndefined; private _prepareAsync; + private _updatePrep; /** * The amount of time in ms to prepare for validation. */ get prepTime(): number; checkText(range: SimpleRange, _text: string, _scope: string[]): ValidationIssue[]; get document(): TextDocument; + updateDocumentText(text: string): void; private addPossibleError; private catchError; private errorCatcherWrapper; + private suggest; + private genSuggestions; } declare type Offset = number; declare type SimpleRange = readonly [Offset, Offset]; From 7a3c29f36c513929e655a7ec0758c8cb2fee0045 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Wed, 23 Mar 2022 20:28:26 +0100 Subject: [PATCH 15/15] Update Memorizer.test.ts --- .../cspell-lib/src/util/Memorizer.test.ts | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/cspell-lib/src/util/Memorizer.test.ts b/packages/cspell-lib/src/util/Memorizer.test.ts index 3df52c16756..f6ffbc14fcb 100644 --- a/packages/cspell-lib/src/util/Memorizer.test.ts +++ b/packages/cspell-lib/src/util/Memorizer.test.ts @@ -1,4 +1,4 @@ -import { memorizer, memorizeLastCall, callOnce } from './Memorizer'; +import { memorizer, memorizeLastCall, callOnce, memorizerAll } from './Memorizer'; describe('Validate Memorizer', () => { test('the memorizer works', () => { @@ -130,3 +130,24 @@ describe('callOnce', () => { expect(calls).toBe(1); }); }); + +describe('memorizerAll', () => { + it('memorizerAll', () => { + function echo(...a: (string | number | undefined)[]): (string | number | undefined)[] { + return a; + } + + const mock = jest.fn(echo); + const fn = memorizerAll(mock); + expect(fn('a')).toEqual(['a']); + expect(fn('b')).toEqual(['b']); + expect(fn('a', 'b')).toEqual(['a', 'b']); + expect(fn('a', 'b')).toEqual(['a', 'b']); + expect(fn(undefined)).toEqual([undefined]); + expect(fn('b')).toEqual(['b']); + expect(fn(undefined)).toEqual([undefined]); + expect(fn('a')).toEqual(['a']); + expect(mock).toHaveBeenCalledTimes(4); + expect(mock).toHaveBeenLastCalledWith(undefined); + }); +});