From 6441f4b41fe0e8b8188fa4c08999450c8958b6f0 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Fri, 30 Sep 2022 13:06:21 +0200 Subject: [PATCH] feat: Use cspell-dictionary Module (#3686) --- .../SpellingDictionaryFromTrie.test.ts | 11 +- .../src/SpellingDictionary/index.ts | 2 +- .../src/__snapshots__/index.test.ts.snap | 1 + packages/cspell-dictionary/src/index.ts | 1 + packages/cspell-lib/api/api.d.ts | 50 +-- .../src/SpellingDictionary/Dictionaries.ts | 15 +- .../DictionaryController/DictionaryLoader.ts | 28 +- .../SpellingDictionary/DictionaryLoader.ts | 2 +- .../SpellingDictionary/SpellingDictionary.ts | 11 +- .../SpellingDictionary.test.ts | 234 ---------- .../SpellingDictionary.ts | 11 - .../SpellingDictionaryCollection.test.ts | 411 ------------------ .../SpellingDictionaryCollection.ts | 189 -------- .../SpellingDictionaryFromTrie.ts | 228 ---------- .../SpellingDictionaryMethods.test.ts | 15 - .../SpellingDictionaryMethods.ts | 136 ------ .../SpellingDictionaryLibOld/charset.ts | 12 - .../createSpellingDictionary.test.ts | 136 ------ .../createSpellingDictionary.ts | 117 ----- .../SpellingDictionaryLibOld/index.ts | 17 - .../src/textValidation/isWordValid.test.ts | 3 +- .../src/textValidation/textValidator.test.ts | 8 +- .../src/textValidation/textValidator.ts | 2 +- 23 files changed, 57 insertions(+), 1583 deletions(-) delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionary.test.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionary.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryCollection.test.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryCollection.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryFromTrie.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryMethods.test.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryMethods.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/charset.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary.test.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary.ts delete mode 100644 packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/index.ts diff --git a/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts b/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts index c11ae70112a..d65f23e9c7e 100644 --- a/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts +++ b/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts @@ -3,14 +3,15 @@ import { __testing__ } from './SpellingDictionaryFromTrie'; const { outerWordForms } = __testing__; -// cspell:ignore guenstig günstig +// cspell:ignore guenstig günstig Bundesstaat Bundeßtaat describe('SpellingDictionaryFromTrie', () => { test.each` - word | repMap | expected - ${'hello'} | ${undefined} | ${['hello']} - ${'guenstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['guenstig', 'günstig']} - ${'günstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['günstig', 'günstig'.normalize('NFD')]} + word | repMap | expected + ${'hello'} | ${undefined} | ${['hello']} + ${'guenstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['guenstig', 'günstig']} + ${'günstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['günstig', 'günstig'.normalize('NFD')]} + ${'Bundesstaat'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['Bundesstaat', 'Bundeßtaat']} `('outerWordForms $word', ({ word, repMap, expected }) => { const mapWord = createMapper(repMap); expect(outerWordForms(word, mapWord ?? ((a) => a))).toEqual(new Set(expected)); diff --git a/packages/cspell-dictionary/src/SpellingDictionary/index.ts b/packages/cspell-dictionary/src/SpellingDictionary/index.ts index 45319d14561..f6b91e17b76 100644 --- a/packages/cspell-dictionary/src/SpellingDictionary/index.ts +++ b/packages/cspell-dictionary/src/SpellingDictionary/index.ts @@ -1,5 +1,5 @@ export { CachingDictionary, createCachingDictionary } from './CachingDictionary'; -export { createSpellingDictionary } from './createSpellingDictionary'; +export { createSpellingDictionary, createFailedToLoadDictionary } from './createSpellingDictionary'; export { createForbiddenWordsDictionary } from './ForbiddenWordsDictionary'; export { createIgnoreWordsDictionary } from './IgnoreWordsDictionary'; export type { diff --git a/packages/cspell-dictionary/src/__snapshots__/index.test.ts.snap b/packages/cspell-dictionary/src/__snapshots__/index.test.ts.snap index afafa3c1b69..57ac07e50f5 100644 --- a/packages/cspell-dictionary/src/__snapshots__/index.test.ts.snap +++ b/packages/cspell-dictionary/src/__snapshots__/index.test.ts.snap @@ -4,6 +4,7 @@ exports[`index verify api 1`] = ` [ "createCachingDictionary", "createCollection", + "createFailedToLoadDictionary", "createForbiddenWordsDictionary", "createIgnoreWordsDictionary", "createSpellingDictionary", diff --git a/packages/cspell-dictionary/src/index.ts b/packages/cspell-dictionary/src/index.ts index 89a2b858e44..163d3348c3e 100644 --- a/packages/cspell-dictionary/src/index.ts +++ b/packages/cspell-dictionary/src/index.ts @@ -1,6 +1,7 @@ export { createCachingDictionary, createCollection, + createFailedToLoadDictionary, createForbiddenWordsDictionary, createIgnoreWordsDictionary, createSpellingDictionary, diff --git a/packages/cspell-lib/api/api.d.ts b/packages/cspell-lib/api/api.d.ts index b97eb15114c..66b213c8300 100644 --- a/packages/cspell-lib/api/api.d.ts +++ b/packages/cspell-lib/api/api.d.ts @@ -2,9 +2,9 @@ import { Glob, CSpellSettingsWithSourceTrace, AdvancedCSpellSettingsWithSourceTrace, Parser, DictionaryDefinitionPreferred, DictionaryDefinitionAugmented, DictionaryDefinitionCustom, TextOffset, TextDocumentOffset, PnPSettings as PnPSettings$1, ImportFileRef, CSpellUserSettings, Issue, MappedText, ParsedText, LocaleId, CSpellSettings } from '@cspell/cspell-types'; export * from '@cspell/cspell-types'; import * as cspellDictModule from 'cspell-dictionary'; -import { SpellingDictionary, SpellingDictionaryOptions, HasOptions, FindResult, SuggestOptions, CachingDictionary, SpellingDictionaryCollection as SpellingDictionaryCollection$1, SuggestionResult as SuggestionResult$1 } from 'cspell-dictionary'; +import { CachingDictionary, SpellingDictionaryCollection, SuggestionResult } from 'cspell-dictionary'; export { SpellingDictionary, SpellingDictionaryCollection, SuggestOptions, SuggestionCollector, SuggestionResult } from 'cspell-dictionary'; -import { CompoundWordsMethod, SuggestionResult, SuggestionCollector, WeightMap } from 'cspell-trie-lib'; +import { WeightMap } from 'cspell-trie-lib'; export { CompoundWordsMethod } from 'cspell-trie-lib'; export { asyncIterableToArray, readFile, readFileSync, writeToFile, writeToFileIterable, writeToFileIterableP } from 'cspell-io'; import { URI } from 'vscode-uri'; @@ -115,35 +115,8 @@ declare namespace index_link_d { }; } -declare function identityString(w: string): string; -declare class SpellingDictionaryCollection implements SpellingDictionary { - readonly dictionaries: SpellingDictionary[]; - readonly name: string; - readonly options: SpellingDictionaryOptions; - readonly mapWord: typeof identityString; - readonly type = "SpellingDictionaryCollection"; - readonly source: string; - readonly isDictionaryCaseSensitive: boolean; - readonly containsNoSuggestWords: boolean; - constructor(dictionaries: SpellingDictionary[], name: string); - has(word: string, hasOptions?: HasOptions): boolean; - find(word: string, hasOptions?: HasOptions): FindResult | undefined; - isNoSuggestWord(word: string, options?: HasOptions): boolean; - isForbidden(word: string): boolean; - suggest(word: string, numSuggestions?: number, compoundMethod?: CompoundWordsMethod, numChanges?: number, ignoreCase?: boolean): SuggestionResult[]; - suggest(word: string, suggestOptions: SuggestOptions): SuggestionResult[]; - _suggest(word: string, suggestOptions: SuggestOptions): SuggestionResult[]; - get size(): number; - genSuggestions(collector: SuggestionCollector, suggestOptions: SuggestOptions): void; - getErrors(): Error[]; - private _isForbiddenInDict; - private _isNoSuggestWord; -} -declare function createCollection$1(dictionaries: SpellingDictionary[], name: string): SpellingDictionaryCollection; - -interface IterableLike { - [Symbol.iterator]: () => Iterator | IterableIterator; -} +declare const createSpellingDictionary: typeof cspellDictModule.createSpellingDictionary; +declare const createCollection: typeof cspellDictModule.createCollection; /** * The keys of an object where the values cannot be undefined. @@ -179,6 +152,8 @@ interface DictionaryDefinitionInternal extends Readonly; + declare type LoadOptions = DictionaryDefinitionInternal; declare class SpellingDictionaryLoadError extends Error { @@ -190,13 +165,6 @@ declare class SpellingDictionaryLoadError extends Error { } declare function isSpellingDictionaryLoadError(e: Error): e is SpellingDictionaryLoadError; -declare function createSpellingDictionary$1(wordList: readonly string[] | IterableLike, name: string, source: string, options: SpellingDictionaryOptions | undefined): SpellingDictionary; - -declare const createSpellingDictionary: typeof createSpellingDictionary$1 | typeof cspellDictModule.createSpellingDictionary; -declare const createCollection: typeof createCollection$1 | typeof cspellDictModule.createCollection; - -declare function refreshDictionaryCache(maxAge?: number): Promise; - declare function stringToRegExp(pattern: string | RegExp, defaultFlags?: string, forceFlags?: string): RegExp | undefined; declare function splitCamelCaseWordWithOffset(wo: TextOffset): Array; @@ -664,7 +632,7 @@ declare class DocumentValidator { interface Preparations { /** loaded config */ config: CSpellSettingsInternal; - dictionary: SpellingDictionaryCollection$1; + dictionary: SpellingDictionaryCollection; /** configuration after applying in-doc settings */ docSettings: CSpellSettingsInternal; finalSettings: CSpellSettingsInternalFinalized; @@ -788,7 +756,7 @@ declare function fileToDocument(file: string, text: string, languageId?: string, declare function fileToDocument(file: string, text?: string, languageId?: string, locale?: string): Document | DocumentWithText; declare function fileToTextDocument(file: string): Promise; -interface SuggestedWordBase extends SuggestionResult$1 { +interface SuggestedWordBase extends SuggestionResult { dictionaries: string[]; } interface SuggestedWord extends SuggestedWordBase { @@ -900,6 +868,6 @@ interface ResolveFileResult { declare function resolveFile(filename: string, relativeTo: string): ResolveFileResult; declare function clearCachedFiles(): Promise; -declare function getDictionary(settings: CSpellUserSettings): Promise; +declare function getDictionary(settings: CSpellUserSettings): Promise; export { CheckTextInfo, ConfigurationDependencies, CreateTextDocumentParams, DetermineFinalDocumentSettingsResult, Document, DocumentValidator, DocumentValidatorOptions, ENV_CSPELL_GLOB_ROOT, ExcludeFilesGlobMap, ExclusionFunction, exclusionHelper_d as ExclusionHelper, FeatureFlag, FeatureFlags, ImportError, ImportFileRefWithError, IncludeExcludeFlag, IncludeExcludeOptions, index_link_d as Link, Logger, SpellCheckFileOptions, SpellCheckFileResult, SpellingDictionaryLoadError, SuggestedWord, SuggestionError, SuggestionOptions, SuggestionsForWordResult, text_d as Text, TextDocument, TextDocumentLine, TextInfoItem, TraceOptions, TraceResult, UnknownFeatureFlagError, ValidationIssue, calcOverrideSettings, checkFilenameMatchesGlob, checkText, checkTextDocument, clearCachedFiles, clearCachedSettingsFiles, combineTextAndLanguageSettings, combineTextAndLanguageSettings as constructSettingsForText, createSpellingDictionary, createCollection as createSpellingDictionaryCollection, createTextDocument, currentSettingsFileVersion, defaultConfigFilenames, defaultFileName, defaultFileName as defaultSettingsFilename, determineFinalDocumentSettings, extractDependencies, extractImportErrors, fileToDocument, fileToTextDocument, finalizeSettings, getCachedFileSize, getDefaultBundledSettings, getDefaultSettings, getDictionary, getGlobalSettings, getLanguagesForBasename as getLanguageIdsForBaseFilename, getLanguagesForExt, getLogger, getSources, getSystemFeatureFlags, isBinaryFile, isSpellingDictionaryLoadError, loadConfig, loadPnP, loadPnPSync, mergeInDocSettings, mergeSettings, readRawSettings, readSettings, readSettingsFiles, refreshDictionaryCache, resolveFile, searchForConfig, sectionCSpell, setLogger, spellCheckDocument, spellCheckFile, suggestionsForWord, suggestionsForWords, traceWords, traceWordsAsync, updateTextDocument, validateText }; diff --git a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts index 45b0f2830a8..48b359bdf1f 100644 --- a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts +++ b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts @@ -1,8 +1,15 @@ +import { + createCollection, + createForbiddenWordsDictionary, + createIgnoreWordsDictionary, + createSpellingDictionary, + SpellingDictionary, + SpellingDictionaryCollection, +} from 'cspell-dictionary'; import { CSpellSettingsInternal, DictionaryDefinitionInternal } from '../Models/CSpellSettingsInternalDef'; import { calcDictionaryDefsToLoad } from '../Settings/DictionarySettings'; import { isDefined } from '../util/util'; import { loadDictionary, loadDictionarySync, refreshCacheEntries } from './DictionaryLoader'; -import { getSpellDictInterface, SpellingDictionary, SpellingDictionaryCollection } from './SpellingDictionary'; export function loadDictionaryDefs(defsToLoad: DictionaryDefinitionInternal[]): Promise[] { return defsToLoad.map(loadDictionary); @@ -33,8 +40,6 @@ function _getDictionaryInternal( spellDictionaries: SpellingDictionary[] ): SpellingDictionaryCollection { const { words = emptyWords, userWords = emptyWords, flagWords = emptyWords, ignoreWords = emptyWords } = settings; - const { createSpellingDictionary, createIgnoreWordsDictionary, createCollection, createForbiddenWordsDictionary } = - getSpellDictInterface(); const settingsWordsDictionary = createSpellingDictionary(words, '[words]', 'From Settings `words`', { caseSensitive: true, @@ -51,9 +56,7 @@ function _getDictionaryInternal( '[ignoreWords]', 'From Settings `ignoreWords`' ); - const flagWordsDictionary = createForbiddenWordsDictionary(flagWords, '[flagWords]', 'From Settings `flagWords`', { - weightMap: undefined, - }); + const flagWordsDictionary = createForbiddenWordsDictionary(flagWords, '[flagWords]', 'From Settings `flagWords`'); const dictionaries = [ ...spellDictionaries, settingsWordsDictionary, diff --git a/packages/cspell-lib/src/SpellingDictionary/DictionaryController/DictionaryLoader.ts b/packages/cspell-lib/src/SpellingDictionary/DictionaryController/DictionaryLoader.ts index 13af478bb6a..eff9f042979 100644 --- a/packages/cspell-lib/src/SpellingDictionary/DictionaryController/DictionaryLoader.ts +++ b/packages/cspell-lib/src/SpellingDictionary/DictionaryController/DictionaryLoader.ts @@ -1,15 +1,15 @@ import { opConcatMap, opFilter, opMap, pipe } from '@cspell/cspell-pipe/sync'; import type { DictionaryFileTypes } from '@cspell/cspell-types'; -import { CSpellIO, Stats } from 'cspell-io'; -import { DictionaryDefinitionInternal } from '../../Models/CSpellSettingsInternalDef'; -import { toError } from '../../util/errors'; import { createFailedToLoadDictionary, createSpellingDictionary, -} from '../SpellingDictionaryLibOld/createSpellingDictionary'; -import { SpellingDictionary } from '../SpellingDictionaryLibOld/SpellingDictionary'; + createSpellingDictionaryFromTrieFile, + SpellingDictionary, +} from 'cspell-dictionary'; +import { CSpellIO, Stats } from 'cspell-io'; +import { DictionaryDefinitionInternal } from '../../Models/CSpellSettingsInternalDef'; +import { toError } from '../../util/errors'; import { SpellingDictionaryLoadError } from '../SpellingDictionaryError'; -import { createSpellingDictionaryTrie } from '../SpellingDictionaryLibOld/SpellingDictionaryFromTrie'; const MAX_AGE = 10000; @@ -156,7 +156,12 @@ export class DictionaryLoader { private loadEntry(uri: string, options: LoadOptions, now = Date.now()): CacheEntry { options = this.normalizeOptions(uri, options); const pDictionary = load(this.reader, uri, options).catch((e) => - createFailedToLoadDictionary(new SpellingDictionaryLoadError(uri, options, e, 'failed to load')) + createFailedToLoadDictionary( + options.name, + uri, + new SpellingDictionaryLoadError(uri, options, e, 'failed to load'), + options + ) ); const pStat = this.getStat(uri); const pending = Promise.all([pDictionary, pStat]); @@ -201,7 +206,10 @@ export class DictionaryLoader { } catch (e) { const error = toError(e); const dictionary = createFailedToLoadDictionary( - new SpellingDictionaryLoadError(uri, options, error, 'failed to load') + options.name, + uri, + new SpellingDictionaryLoadError(uri, options, error, 'failed to load'), + options ); const pending = Promise.resolve([dictionary, stat] as const); return { @@ -351,10 +359,10 @@ function loadSimpleWordListSync(readLinesSync: ReaderSync, filename: string, opt async function loadTrie(readLines: Reader, filename: string, options: LoadOptions) { const lines = await readLines(filename); - return createSpellingDictionaryTrie(lines, options.name, filename, options); + return createSpellingDictionaryFromTrieFile(lines, options.name, filename, options); } function loadTrieSync(readLinesSync: ReaderSync, filename: string, options: LoadOptions) { const lines = readLinesSync(filename); - return createSpellingDictionaryTrie(lines, options.name, filename, options); + return createSpellingDictionaryFromTrieFile(lines, options.name, filename, options); } diff --git a/packages/cspell-lib/src/SpellingDictionary/DictionaryLoader.ts b/packages/cspell-lib/src/SpellingDictionary/DictionaryLoader.ts index 41e6f025abf..80d6a728d60 100644 --- a/packages/cspell-lib/src/SpellingDictionary/DictionaryLoader.ts +++ b/packages/cspell-lib/src/SpellingDictionary/DictionaryLoader.ts @@ -2,7 +2,7 @@ import { CSpellIO } from 'cspell-io'; import { DictionaryDefinitionInternal } from '../Models/CSpellSettingsInternalDef'; import { getCSpellIO } from '../static'; import { DictionaryLoader } from './DictionaryController'; -import { SpellingDictionary } from './SpellingDictionaryLibOld/SpellingDictionary'; +import { SpellingDictionary } from 'cspell-dictionary'; export type { LoadOptions } from './DictionaryController'; let loader: DictionaryLoader | undefined; diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts index 3308dc4d427..f294a141efb 100644 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts @@ -1,6 +1,4 @@ import * as cspellDictModule from 'cspell-dictionary'; -import { getSystemFeatureFlags } from '../FeatureFlags'; -import { SpellingDictionaryLibOld } from './SpellingDictionaryLibOld'; export { CompoundWordsMethod } from 'cspell-trie-lib'; const SpellingDictionaryModule = { @@ -8,12 +6,10 @@ const SpellingDictionaryModule = { createForbiddenWordsDictionary: cspellDictModule.createForbiddenWordsDictionary, createSpellingDictionary: cspellDictModule.createSpellingDictionary, createIgnoreWordsDictionary: cspellDictModule.createIgnoreWordsDictionary, + createSpellingDictionaryFromTrieFile: cspellDictModule.createSpellingDictionaryFromTrieFile, } as const; -type SpellDictInterface = typeof SpellingDictionaryModule | typeof SpellingDictionaryLibOld; - -const flagUseCSpellDictionary = 'use-cspell-dictionary'; -getSystemFeatureFlags().register(flagUseCSpellDictionary, 'Use the CSpell Dictionary module.'); +type SpellDictInterface = typeof SpellingDictionaryModule; export type { FindOptions, @@ -29,8 +25,7 @@ export type { } from 'cspell-dictionary'; export function getSpellDictInterface(): SpellDictInterface { - const useModule = getSystemFeatureFlags().getFlagBool(flagUseCSpellDictionary) ?? true; - return useModule ? SpellingDictionaryModule : SpellingDictionaryLibOld; + return SpellingDictionaryModule; } export const createSpellingDictionary = getSpellDictInterface().createSpellingDictionary; diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionary.test.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionary.test.ts deleted file mode 100644 index 21356f06ed1..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionary.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { __testMethods__ } from './SpellingDictionaryMethods'; -import { createSpellingDictionary } from './createSpellingDictionary'; -import { SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie'; -import { Trie } from 'cspell-trie-lib'; -import { SpellingDictionaryOptions } from './SpellingDictionary'; - -// cSpell:ignore aple - -describe('Verify building Dictionary', () => { - test('build from word list', async () => { - const words = ['apple', 'ape', 'able', 'apple', 'banana', 'orange', 'pear', 'aim', 'approach', 'bear']; - - const dict = await createSpellingDictionary(words, 'words', 'test', opts()); - expect(dict.name).toBe('words'); - expect(dict.has('apple')).toBe(true); - const suggestions = dict.suggest('aple').map(({ word }) => word); - expect(suggestions).toEqual(expect.arrayContaining(['apple'])); - expect(suggestions).toEqual(expect.arrayContaining(['ape'])); - expect(suggestions).toEqual(expect.not.arrayContaining(['banana'])); - }); - - test('compounds from word list', async () => { - const words = [ - 'apple', - 'apples', - 'ape', - 'able', - 'apple', - 'banana', - 'orange', - 'pear', - 'aim', - 'approach', - 'bear', - ]; - - const dict = await createSpellingDictionary(words, 'words', 'test', opts({ useCompounds: true })); - expect(dict.has('apple')).toBe(true); - expect(dict.has('Apple')).toBe(true); - expect(dict.has('APPLE')).toBe(true); - expect(dict.has('APPLEs')).toBe(true); - expect(dict.has('APPles')).toBe(true); // cspell:disable-line - // cspell:ignore applebanana applebananas applebananaorange - expect(dict.has('applebanana')).toBe(true); - expect(dict.has('applebananaorange')).toBe(true); - expect(dict.has('applebananas')).toBe(false); - }); - - test('case-sensitive word list', async () => { - const words = ['apple', 'Seattle', 'Amsterdam', 'surf', 'words', 'English', 'McGreyer']; - - const dict = await createSpellingDictionary( - words, - 'words', - 'test', - opts({ - caseSensitive: true, - }) - ); - const ignoreCase = { ignoreCase: true }; - const useCase = { ignoreCase: false }; - expect(dict.has('apple', useCase)).toBe(true); - expect(dict.has('Apple', ignoreCase)).toBe(true); - expect(dict.has('Apple', useCase)).toBe(true); - expect(dict.has('APPLE', useCase)).toBe(true); - expect(dict.has('Seattle', useCase)).toBe(true); - expect(dict.has('seattle', useCase)).toBe(false); - expect(dict.has('English', useCase)).toBe(true); - expect(dict.has('english', useCase)).toBe(false); - expect(dict.has('ENGLISH', useCase)).toBe(true); - expect(dict.has('McGreyer', useCase)).toBe(true); - expect(dict.has('mcgreyer', useCase)).toBe(false); // cspell:disable-line - // We do not support mixed case as all caps matching at this point. - expect(dict.has('MCGREYER', useCase)).toBe(false); // cspell:disable-line - expect(dict.has('MCGREYER', ignoreCase)).toBe(true); // cspell:disable-line - }); - - test('Suggest Trie', () => { - const words = [ - 'apple', - 'ape', - 'able', - 'apple', - 'banana', - 'orange', - 'pear', - 'aim', - 'approach', - 'bear', - 'cattle', - 'rattle', - 'battle', - 'rattles', - 'battles', - 'tattles', - ]; - const trie = Trie.create(words); - const dict = new SpellingDictionaryFromTrie(trie, 'trie', opts()); - // cspell:ignore cattles - const results = dict.suggest('Cattles'); - const suggestions = results.map(({ word }) => word); - expect(suggestions).toEqual(['cattle', 'battles', 'rattles', 'tattles', 'battle', 'rattle']); - expect(suggestions).toEqual(expect.not.arrayContaining(['banana'])); - }); - - test('build from list containing non-strings', async () => { - // eslint-disable-next-line no-sparse-arrays - const words = ['apple', 'ape', 'able', , 'apple', 'banana', 'orange', 5, 'pear', 'aim', 'approach', 'bear']; - - const dict = await createSpellingDictionary(words as string[], 'words', 'test', opts()); - expect(dict.name).toBe('words'); - // expect(dict).toBeInstanceOf(SpellingDictionaryFromTrie); - expect(dict.has('apple')).toBe(true); - const suggestions = dict.suggest('aple').map(({ word }) => word); - expect(suggestions).toEqual(expect.arrayContaining(['apple'])); - expect(suggestions).toEqual(expect.arrayContaining(['ape'])); - expect(suggestions).toEqual(expect.not.arrayContaining(['banana'])); - }); - - type Test = [string, boolean, string[]]; - // cspell:ignore café - const tests: Test[] = [ - ['house', false, ['house']], - ['House', false, ['House', 'house']], - ['café', false, ['cafe', 'café']], - ['Café', false, ['Cafe', 'Café', 'cafe', 'café']], - ['House', true, ['House', '~house']], - ['HOUSE', true, ['HOUSE', '~house']], - ['Café', true, ['Café', '~Cafe', '~cafe', '~café']], - // Make sure all accent forms work. - ['café'.normalize(), false, ['cafe', 'café']], - ['café'.normalize('NFD'), false, ['cafe', 'café']], - ['café'.normalize('NFKC'), false, ['cafe', 'café']], - ['café'.normalize('NFKD'), false, ['cafe', 'café']], - ]; - test.each(tests)( - 'wordDictionaryFormsCollector %s %o %o', - (word: string, isCaseSensitive: boolean, expected: string[]) => { - const collector = __testMethods__.wordDictionaryFormsCollector(isCaseSensitive ? '~' : ''); - expect([...collector(word)].sort()).toEqual(expected.sort()); - expect([...collector(word)]).toEqual([]); - } - ); -}); - -describe('Validate wordSearchForms', () => { - test.each` - word | isCaseSensitive | ignoreCase | expected - ${'house'} | ${false} | ${false} | ${['house']} - ${'House'} | ${false} | ${false} | ${['house']} - ${'House'} | ${false} | ${false} | ${['house']} - ${'House'} | ${true} | ${false} | ${['House', 'house']} - ${'HOUSE'} | ${false} | ${false} | ${['house']} - ${'HOUSE'} | ${true} | ${false} | ${['HOUSE', 'House', 'house']} - ${'café'} | ${false} | ${false} | ${['café']} - ${'café'} | ${true} | ${false} | ${['café']} - ${'café'} | ${true} | ${true} | ${['café']} - ${'Café'} | ${false} | ${false} | ${['café']} - ${'Café'} | ${false} | ${true} | ${['café']} - ${'Café'} | ${true} | ${false} | ${['Café', 'café']} - ${'Café'} | ${true} | ${true} | ${['café']} - ${'CAFÉ'} | ${false} | ${false} | ${['café']} - ${'CAFÉ'} | ${false} | ${true} | ${['café']} - ${'CAFÉ'} | ${true} | ${false} | ${['CAFÉ', 'Café', 'café']} - ${'CAFÉ'} | ${true} | ${true} | ${['café']} - ${'café'.normalize()} | ${false} | ${false} | ${['café']} - ${'café'.normalize('NFD')} | ${false} | ${false} | ${['café']} - ${'café'.normalize('NFKC')} | ${false} | ${false} | ${['café']} - ${'café'.normalize('NFKD')} | ${false} | ${false} | ${['café']} - `('$word $isCaseSensitive $ignoreCase $expected', ({ word, isCaseSensitive, ignoreCase, expected }) => { - const words = __testMethods__.wordSearchFormsArray(word, isCaseSensitive, ignoreCase); - expect(words.sort()).toEqual(expected.sort()); - }); -}); - -describe('Verify Case Sensitive Dictionaries', () => { - test.each` - word | ignoreCase | expected - ${'Paris'} | ${undefined} | ${true} - ${'PARIS'} | ${undefined} | ${true} - ${'paris'} | ${undefined} | ${true} - ${'Paris'} | ${true} | ${true} - ${'PARIS'} | ${true} | ${true} - ${'paris'} | ${true} | ${true} - ${'Paris'} | ${false} | ${true} - ${'PARIS'} | ${false} | ${true} - ${'paris'} | ${false} | ${false} - ${'Köln'} | ${false} | ${true} - ${'köln'} | ${false} | ${false} - ${'KÖLN'} | ${false} | ${true} - `(`Has $word Case: $ignoreCase Exp: $expected`, ({ word, ignoreCase, expected }) => { - const dict = sampleDict(); - expect(dict.has(word, { ignoreCase })).toBe(expected); - }); - - // cspell:ignore kuln - test.each` - word | ignoreCase | expected - ${'köln'} | ${false} | ${['Köln']} - ${'köln'} | ${true} | ${['köln', 'koln', 'Köln']} - ${'koln'} | ${true} | ${['koln', 'köln', 'Köln']} - ${'kuln'} | ${false} | ${['Köln']} - ${'kuln'} | ${true} | ${['koln', 'köln', 'Köln']} - `('Suggestions for $word $ignoreCase $expected', ({ word, ignoreCase, expected }) => { - // cspell:ignore koln - const dict = sampleDict(); - const sugs = dict.suggest(word, { ignoreCase }); - const sugWords = sugs.map((s) => s.word); - expect(sugWords).toEqual(expected); - }); -}); - -function sampleDict() { - const words = sampleWords(); - return createSpellingDictionary(words, 'words', 'test', opts({ caseSensitive: true })); -} - -// cspell:words métro Rhône Köln Düsseldorf -function sampleWords() { - return ` - England Canada Netherlands France German China Belgium - Paris Chicago Amsterdam Antwerp Brussels Rhône Cologne Köln Düsseldorf - métro cafe café metro - apple apples ape apes around astound profound compound - table tables tabled - `.split(/\s+/g); -} - -function opts(opts: Partial = {}): SpellingDictionaryOptions { - return { - weightMap: undefined, - ...opts, - }; -} diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionary.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionary.ts deleted file mode 100644 index 079484db77c..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionary.ts +++ /dev/null @@ -1,11 +0,0 @@ -export { - FindOptions, - FindResult, - HasOptions, - SearchOptions, - SpellingDictionary, - SpellingDictionaryOptions, - SuggestOptions, -} from 'cspell-dictionary'; -export { CompoundWordsMethod } from 'cspell-trie-lib'; -export type { SuggestionCollector, SuggestionResult } from 'cspell-trie-lib'; diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryCollection.test.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryCollection.test.ts deleted file mode 100644 index 63628f30692..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryCollection.test.ts +++ /dev/null @@ -1,411 +0,0 @@ -import * as Trie from 'cspell-trie-lib'; -import { mapDictDefToInternal } from '../../Settings/DictionarySettings'; -import { - createFailedToLoadDictionary, - createForbiddenWordsDictionary, - createSpellingDictionary, -} from './createSpellingDictionary'; -import { CompoundWordsMethod, SpellingDictionaryOptions } from './SpellingDictionary'; -import { createCollection } from './SpellingDictionaryCollection'; -import { SpellingDictionaryLoadError } from '../SpellingDictionaryError'; -import { SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie'; - -const di = mapDictDefToInternal; - -describe('Verify using multiple dictionaries', () => { - const wordsA = [ - '', - 'apple', - 'banana', - 'orange', - 'pear', - 'pineapple', - 'mango', - 'avocado', - 'grape', - 'strawberry', - 'blueberry', - 'blackberry', - ]; - const wordsB = ['ape', 'lion', 'tiger', 'elephant', 'monkey', 'gazelle', 'antelope', 'aardvark', 'hyena']; - const wordsC = ['ant', 'snail', 'beetle', 'worm', 'stink bug', 'centipede', 'millipede', 'flea', 'fly']; - const wordsD = ['red*', 'green*', 'blue*', 'pink*', 'black*', '*berry', '+-fruit', '*bug', 'pinkie']; - const wordsF = ['!pink*', '+berry', '+bug', '!stinkbug']; - const wordsG = ['café', 'accent']; - - const wordsLegacy = ['error', 'code', 'system', 'ctrl']; - - // cspell:ignore pinkberry behaviour colour - const wordsNoSug = ['colour', 'behaviour', 'favour', 'pinkberry']; - - const dictNoSug = createSpellingDictionary(wordsNoSug, 'words-no-suggest', 'test', opts({ noSuggest: true })); - const dictLegacy = createSpellingDictionary(wordsLegacy, 'legacy-dict', 'test', opts({ useCompounds: true })); - - test('checks for existence', async () => { - const dicts = await Promise.all([ - createSpellingDictionary(wordsA, 'wordsA', 'test', opts()), - createSpellingDictionary(wordsB, 'wordsB', 'test', opts()), - createSpellingDictionary(wordsC, 'wordsC', 'test', opts()), - createSpellingDictionary(wordsD, 'wordsD', 'test', opts()), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]); - - const dictCollection = createCollection(dicts, 'test'); - expect(dictCollection.has('mango')).toBe(true); - expect(dictCollection.has('tree')).toBe(false); - expect(dictCollection.has('avocado')).toBe(false); - expect(dictCollection.has('')).toBe(false); - expect(dictCollection.has('red-fruit')).toBe(true); - expect(dictCollection.has('-fruit')).toBe(false); - expect(dictCollection.has('blackberry')).toBe(true); - expect(dictCollection.size).toBeGreaterThanOrEqual(wordsA.length - 1 + wordsB.length + wordsC.length); - }); - - test('checks mapWord is identity', async () => { - const dicts = await Promise.all([createSpellingDictionary(wordsA, 'wordsA', 'test', opts())]); - - const dictCollection = createCollection(dicts, 'test'); - expect(dictCollection.mapWord('Hello')).toBe('Hello'); - }); - - test('checks for suggestions', async () => { - const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA', opts()); - const dicts = await Promise.all([ - trie, - createSpellingDictionary(wordsB, 'wordsB', 'test', opts()), - createSpellingDictionary(wordsA, 'wordsA', 'test', opts()), - createSpellingDictionary(wordsC, 'wordsC', 'test', opts()), - createFailedToLoadDictionary( - new SpellingDictionaryLoadError( - './missing.txt', - di({ name: 'error', path: './missing.txt' }, __filename), - new Error('error'), - 'failed to load' - ) - ), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]); - - const dictCollection = createCollection(dicts, 'test'); - expect(dictCollection.getErrors?.()).toHaveLength(1); - const sugsForTango = dictCollection.suggest('tango', 10); - expect(sugsForTango).toHaveLength(1); - expect(sugsForTango[0].word).toEqual('mango'); - // make sure there is only one mango suggestion. - expect(sugsForTango.map((a) => a.word).filter((a) => a === 'mango')).toEqual(['mango']); - }); - - test('checks for compound suggestions', async () => { - // Add "wordsA" twice, once as a compound dictionary and once as a normal dictionary. - const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA', opts()); - trie.options.useCompounds = true; - const dicts = await Promise.all([ - trie, - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]); - - // cspell:ignore appletango applemango - const dictCollection = createCollection(dicts, 'test'); - const sugResult = dictCollection.suggest('appletango', 10, CompoundWordsMethod.SEPARATE_WORDS); - const sugs = sugResult.map((a) => a.word); - expect(sugs).not.toContain('apple+mango'); - expect(sugs).toContain('apple mango'); - }); - - test('checks for compound NONE suggestions', async () => { - // Add "wordsA" twice, once as a compound dictionary and once as a normal dictionary. - const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA', opts()); - trie.options.useCompounds = true; - const dicts = await Promise.all([ - trie, - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]); - - // cspell:ignore appletango applemango - const dictCollection = createCollection(dicts, 'test'); - const sugResult = dictCollection.suggest('applemango', 10, CompoundWordsMethod.NONE); - const sugs = sugResult.map((a) => a.word); - expect(sugs).not.toContain('apple+mango'); - expect(sugs).not.toContain('apple mango'); - expect(sugs).toContain('apple'); - expect(sugs).toContain('mango'); - }); - - test('checks for compound JOIN_WORDS suggestions', async () => { - // Add "wordsA" twice, once as a compound dictionary and once as a normal dictionary. - const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA', opts()); - trie.options.useCompounds = true; - const dicts = await Promise.all([ - trie, - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]); - - // cspell:ignore appletango applemango - const dictCollection = createCollection(dicts, 'test'); - const sugResult = dictCollection.suggest('applemango', 10, CompoundWordsMethod.JOIN_WORDS); - const sugs = sugResult.map((a) => a.word); - expect(sugs).toContain('apple+mango'); - expect(sugs).not.toContain('apple mango'); - // possible word combinations - expect(sugs).toContain('apple'); - expect(sugs).toContain('apple+apple'); - expect(sugs).toContain('grape+mango'); - }); - - test('checks for compound suggestions with numbChanges', async () => { - const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA', opts()); - const dicts = await Promise.all([ - trie, - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]); - - // cspell:ignore appletango applemango - const dictCollection = createCollection(dicts, 'test'); - const sugResult = dictCollection.suggest('appletango', 10, CompoundWordsMethod.SEPARATE_WORDS, 2); - const sugs = sugResult.map((a) => a.word); - expect(sugs).toHaveLength(1); - expect(sugs).not.toContain('apple+mango'); - expect(sugs).toContain('apple mango'); - }); - - test.each` - word | expected - ${'redberry'} | ${true} - ${'pink'} | ${false} - ${'bug'} | ${true} - ${'blackberry'} | ${true} - ${'pinkbug'} | ${true} - ${'cafe'} | ${false} - ${'café'} | ${true} - ${'cafe\u0301'} | ${true} - ${'accent'} | ${true} - ${'áccent'} | ${true /* ignore the accent. cspell:disable-line */} - ${'a\u0301ccent'} | ${true /* ignore the accent. cspell:disable-line */} - ${'applé'} | ${true /* ignore the accent. cspell:disable-line */} - `('checks has word: "$word"', ({ word, expected }) => { - const dicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test', { dictionaryInformation: { ignore: '\u0300-\u0362' } }), - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), - createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), - createSpellingDictionary(wordsG, 'wordsA', 'test', { - dictionaryInformation: { ignore: '\u0300-\u0362' }, - caseSensitive: true, - }), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]; - - const dictCollection = createCollection(dicts, 'test'); - expect(dictCollection.has(word, { ignoreCase: false })).toEqual(expected); - }); - - test.each` - word | expected - ${'redberry'} | ${{ found: 'redberry', forbidden: false, noSuggest: false }} - ${'pinkberry'} | ${{ found: 'pinkberry', forbidden: false, noSuggest: true }} - ${'pink'} | ${{ found: 'pink', forbidden: true, noSuggest: false }} - ${'bug'} | ${{ found: 'bug', forbidden: false, noSuggest: false }} - ${'blackberry'} | ${{ found: 'blackberry', forbidden: false, noSuggest: false }} - ${'pinkbug'} | ${{ found: 'pinkbug', forbidden: false, noSuggest: false }} - ${'colour'} | ${{ found: 'colour', forbidden: false, noSuggest: true }} - ${'behaviour'} | ${{ found: 'behaviour', forbidden: false, noSuggest: true }} - `('find: "$word"', ({ word, expected }) => { - const dicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), - createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - dictNoSug, - ]; - - const dictCollection = createCollection(dicts, 'test'); - expect(dictCollection.find(word)).toEqual(expected); - }); - - // cspell:ignore error* *code ctrl* *code *berry* - test.each` - word | expected - ${'redberry'} | ${{ found: 'redberry', forbidden: false, noSuggest: false }} - ${'pinkberry'} | ${{ found: 'pinkberry', forbidden: false, noSuggest: true }} - ${'berryberry'} | ${{ found: 'berry+berry', forbidden: false, noSuggest: false }} - ${'errorcode'} | ${{ found: 'error+code', forbidden: false, noSuggest: false }} - ${'ctrlcode'} | ${{ found: 'ctrl+code', forbidden: false, noSuggest: false }} - ${'pink'} | ${{ found: 'pink', forbidden: true, noSuggest: false }} - ${'bug'} | ${{ found: 'bug', forbidden: false, noSuggest: false }} - ${'blackberry'} | ${{ found: 'blackberry', forbidden: false, noSuggest: false }} - ${'pinkbug'} | ${{ found: 'pinkbug', forbidden: false, noSuggest: false }} - ${'colour'} | ${{ found: 'colour', forbidden: false, noSuggest: true }} - ${'behaviour'} | ${{ found: 'behaviour', forbidden: false, noSuggest: true }} - `('find compound: "$word"', ({ word, expected }) => { - const dicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), - createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - dictNoSug, - dictLegacy, - ]; - - const dictCollection = createCollection(dicts, 'test'); - expect(dictCollection.find(word, { useCompounds: true })).toEqual(expected); - }); - - // cspell:ignore pinkbug redberry - // Note: `pinkbug` is not forbidden because compound forbidden words is not yet supported. - test.each` - word | expected - ${'redberry'} | ${false} - ${'pink'} | ${true} - ${'bug'} | ${false} - ${'blackberry'} | ${false} - ${'stinkbug'} | ${true} - ${'pinkbug'} | ${false} - `('checks forbid word: "$word"', ({ word, expected }) => { - const dicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), - createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]; - - const dictCollection = createCollection(dicts, 'test'); - expect(dictCollection.isForbidden(word)).toEqual(expected); - }); - - function sr(word: string, cost: number) { - return { word, cost }; - } - - test.each` - word | expected - ${'redberry'} | ${[sr('redberry', 0), sr('red berry', 105)]} - ${'pink'} | ${[sr('pinkie', 189)]} - ${'bug'} | ${[sr('bug', 5)]} - ${'blackberry'} | ${[sr('blackberry', 0), sr('black berry', 98)]} - ${'stinkbug'} | ${[sr('stink bug', 103), sr('pinkbug', 198)]} - `('checks suggestions word: "$word"', ({ word, expected }) => { - const dicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), - createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]; - - const dictCollection = createCollection(dicts, 'test'); - expect(dictCollection.suggest(word, 2)).toEqual(expected); - }); - - test('checks for suggestions with flagged words', async () => { - const dicts = await Promise.all([ - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]); - - const dictCollection = createCollection(dicts, 'test'); - const sugs = dictCollection.suggest('avocado', 10); - expect(sugs.map((r) => r.word)).not.toContain('avocado'); - }); - - test('checks for suggestions from mixed sources', async () => { - const dicts = await Promise.all([ - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - ]); - - const dictCollection = createCollection(dicts, 'test'); - expect(dictCollection.has('mango')).toBe(true); - expect(dictCollection.has('lion')).toBe(true); - expect(dictCollection.has('ant')).toBe(true); - - const sugsForTango = dictCollection.suggest('tango', 10); - expect(sugsForTango).toHaveLength(1); - expect(sugsForTango[0].word).toBe('mango'); - // make sure there is only one mango suggestion. - expect(sugsForTango.map((a) => a.word).filter((a) => a === 'mango')).toEqual(['mango']); - - // cspell:ignore cellipede - const sugsForCellipede = dictCollection.suggest('cellipede', 5); - expect(sugsForCellipede).toHaveLength(2); - expect(sugsForCellipede.map((s) => s.word)).toContain('centipede'); - expect(sugsForCellipede.map((s) => s.word)).toContain('millipede'); - }); -}); - -describe('Validate looking up words', () => { - const wordsA = [ - '', - 'apple', - 'banana', - 'orange', - 'pear', - 'pineapple', - 'mango', - 'avocado', - 'grape', - 'strawberry', - 'blueberry', - 'blackberry', - ]; - const wordsB = ['ape', 'lion', 'tiger', 'elephant', 'monkey', 'gazelle', 'antelope', 'aardvark', 'hyena']; - const wordsC = ['ant', 'snail', 'beetle', 'worm', 'stink bug', 'centipede', 'millipede', 'flea', 'fly']; - const wordsD = ['red*', 'green*', 'blue*', 'pink*', 'black*', '*berry', '+-fruit']; - const cities = ['Seattle', 'Berlin', 'Amsterdam', 'Rome', 'London', 'Mumbai', 'Tokyo']; - - const testDicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), - createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), - createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), - createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), - createSpellingDictionary(cities, 'cities', 'test', undefined), - createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), - ]; - - const testDictCollection = createCollection(testDicts, 'test'); - - interface HasWordTest { - word: string; - found: boolean; - } - - test.each` - word | found - ${'Amsterdam'} | ${true} - ${'amsterdam'} | ${true} - ${'Black'} | ${true} - ${'black-fruit'} | ${true} - `('Has word "$word" $found', ({ word, found }: HasWordTest) => { - expect(testDictCollection.has(word)).toBe(found); - }); -}); - -function opts(opts: Partial = {}): SpellingDictionaryOptions { - return { - weightMap: undefined, - ...opts, - }; -} diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryCollection.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryCollection.ts deleted file mode 100644 index 154c07a9fae..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryCollection.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { CASE_INSENSITIVE_PREFIX } from 'cspell-trie-lib'; -import { genSequence } from 'gensequence'; -import { getDefaultSettings } from '../../Settings'; -import { memorizer, memorizerKeyBy } from '../../util/Memorizer'; -import { clean, isDefined } from '../../util/util'; -import { - CompoundWordsMethod, - FindResult, - HasOptions, - SearchOptions, - SpellingDictionary, - SpellingDictionaryOptions, - SuggestionCollector, - SuggestionResult, - SuggestOptions, -} from './SpellingDictionary'; -import { SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie'; -import { - defaultNumSuggestions, - hasOptionToSearchOption, - SuggestArgs, - suggestArgsToSuggestOptions, - suggestionCollector, -} from './SpellingDictionaryMethods'; - -function identityString(w: string): string { - return w; -} - -class SpellingDictionaryCollection implements SpellingDictionary { - readonly options: SpellingDictionaryOptions = { weightMap: undefined }; - readonly mapWord = identityString; - readonly type = 'SpellingDictionaryCollection'; - readonly source: string; - readonly isDictionaryCaseSensitive: boolean; - readonly containsNoSuggestWords: boolean; - - constructor(readonly dictionaries: SpellingDictionary[], readonly name: string) { - this.dictionaries = this.dictionaries.sort((a, b) => b.size - a.size); - this.source = dictionaries.map((d) => d.name).join(', '); - this.isDictionaryCaseSensitive = this.dictionaries.reduce((a, b) => a || b.isDictionaryCaseSensitive, false); - this.containsNoSuggestWords = this.dictionaries.reduce((a, b) => a || b.containsNoSuggestWords, false); - } - - public has(word: string, hasOptions?: HasOptions): boolean { - const options = hasOptionToSearchOption(hasOptions); - return !!isWordInAnyDictionary(this.dictionaries, word, options) && !this.isForbidden(word); - } - - public find(word: string, hasOptions?: HasOptions): FindResult | undefined { - const options = hasOptionToSearchOption(hasOptions); - const { - found = false, - forbidden = false, - noSuggest = false, - } = findInAnyDictionary(this.dictionaries, word, options) || {}; - return { found, forbidden, noSuggest }; - } - - public isNoSuggestWord(word: string, options?: HasOptions): boolean { - return this._isNoSuggestWord(word, options); - } - - public isForbidden(word: string): boolean { - return !!this._isForbiddenInDict(word) && !this.isNoSuggestWord(word); - } - - public suggest( - word: string, - numSuggestions?: number, - compoundMethod?: CompoundWordsMethod, - numChanges?: number, - ignoreCase?: boolean - ): SuggestionResult[]; - public suggest(word: string, suggestOptions: SuggestOptions): SuggestionResult[]; - public suggest(...args: SuggestArgs): SuggestionResult[] { - const [word] = args; - const suggestOptions = suggestArgsToSuggestOptions(args); - return this._suggest(word, suggestOptions); - } - - public _suggest(word: string, suggestOptions: SuggestOptions): SuggestionResult[] { - const { - numSuggestions = getDefaultSettings(false).numSuggestions || defaultNumSuggestions, - numChanges, - ignoreCase, - includeTies, - timeout, - } = suggestOptions; - const prefixNoCase = CASE_INSENSITIVE_PREFIX; - const filter = (word: string, _cost: number) => { - return ( - (ignoreCase || word[0] !== prefixNoCase) && - !this.isForbidden(word) && - !this.isNoSuggestWord(word, suggestOptions) - ); - }; - const collector = suggestionCollector( - word, - clean({ - numSuggestions, - filter, - changeLimit: numChanges, - includeTies, - ignoreCase, - timeout, - }) - ); - this.genSuggestions(collector, suggestOptions); - return collector.suggestions.map((r) => ({ ...r, word: r.word })); - } - - public get size(): number { - return this.dictionaries.reduce((a, b) => a + b.size, 0); - } - - public genSuggestions(collector: SuggestionCollector, suggestOptions: SuggestOptions): void { - const _suggestOptions = { ...suggestOptions }; - const { compoundMethod = CompoundWordsMethod.SEPARATE_WORDS } = suggestOptions; - _suggestOptions.compoundMethod = this.options.useCompounds ? CompoundWordsMethod.JOIN_WORDS : compoundMethod; - this.dictionaries.forEach((dict) => dict.genSuggestions(collector, _suggestOptions)); - } - - public getErrors(): Error[] { - return this.dictionaries.reduce((errors, dict) => errors.concat(dict.getErrors?.() || []), [] as Error[]); - } - - private _isForbiddenInDict = memorizer( - (word: string) => isWordForbiddenInAnyDictionary(this.dictionaries, word), - SpellingDictionaryFromTrie.cachedWordsLimit - ); - - private _isNoSuggestWord = memorizerKeyBy( - (word: string, options?: HasOptions) => { - if (!this.containsNoSuggestWords) return false; - return !!isNoSuggestWordInAnyDictionary(this.dictionaries, word, options || {}); - }, - (word: string, options?: HasOptions) => { - const opts = hasOptionToSearchOption(options); - return [word, opts.useCompounds, opts.ignoreCase].join(); - }, - SpellingDictionaryFromTrie.cachedWordsLimit - ); -} - -export const SpellingDictionaryCollectionLegacy = SpellingDictionaryCollection; - -export function createCollection(dictionaries: SpellingDictionary[], name: string): SpellingDictionaryCollection { - return new SpellingDictionaryCollection(dictionaries, name); -} - -function isWordInAnyDictionary( - dicts: SpellingDictionary[], - word: string, - options: SearchOptions -): SpellingDictionary | undefined { - return genSequence(dicts).first((dict) => dict.has(word, options)); -} - -function findInAnyDictionary( - dicts: SpellingDictionary[], - word: string, - options: SearchOptions -): FindResult | undefined { - const found = dicts.map((dict) => dict.find(word, options)).filter(isDefined); - if (!found.length) return undefined; - return found.reduce((a, b) => ({ - found: a.forbidden ? a.found : b.forbidden ? b.found : a.found || b.found, - forbidden: a.forbidden || b.forbidden, - noSuggest: a.noSuggest || b.noSuggest, - })); -} - -function isNoSuggestWordInAnyDictionary( - dicts: SpellingDictionary[], - word: string, - options: HasOptions -): SpellingDictionary | undefined { - return genSequence(dicts).first((dict) => dict.isNoSuggestWord(word, options)); -} - -function isWordForbiddenInAnyDictionary(dicts: SpellingDictionary[], word: string): SpellingDictionary | undefined { - return genSequence(dicts).first((dict) => dict.isForbidden(word)); -} - -export const __testing__ = { - isWordInAnyDictionary, - isWordForbiddenInAnyDictionary, -}; diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryFromTrie.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryFromTrie.ts deleted file mode 100644 index 2abaa393998..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryFromTrie.ts +++ /dev/null @@ -1,228 +0,0 @@ -import type { - FindFullResult, - FindWordOptions, - SuggestionCollector, - SuggestionResult, - WeightMap, -} from 'cspell-trie-lib'; -import { CompoundWordsMethod, importTrie, suggestionCollector, Trie } from 'cspell-trie-lib'; -import { getDefaultSettings } from '../../Settings'; -import { memorizer } from '../../util/Memorizer'; -import { createMapper } from '../../util/repMap'; -import { clean } from '../../util/util'; -import { charsetToRegExp } from './charset'; -import { - FindResult, - HasOptions, - SpellingDictionary, - SpellingDictionaryOptions, - SuggestOptions, -} from './SpellingDictionary'; -import { - createWeightMapFromDictionaryInformation, - defaultNumSuggestions, - hasOptionToSearchOption, - impersonateCollector, - SuggestArgs, - suggestArgsToSuggestOptions, - wordSearchForms, - wordSuggestFormsArray, -} from './SpellingDictionaryMethods'; -export class SpellingDictionaryFromTrie implements SpellingDictionary { - static readonly cachedWordsLimit = 50000; - private _size = 0; - readonly knownWords = new Set(); - readonly unknownWords = new Set(); - readonly mapWord: (word: string) => string; - readonly type = 'SpellingDictionaryFromTrie'; - readonly isDictionaryCaseSensitive: boolean; - readonly containsNoSuggestWords: boolean; - readonly ignoreCharactersRegExp: RegExp | undefined; - - private weightMap: WeightMap | undefined; - - constructor( - readonly trie: Trie, - readonly name: string, - readonly options: SpellingDictionaryOptions, - readonly source = 'from trie', - size?: number - ) { - this.mapWord = createMapper(options.repMap || []); - this.isDictionaryCaseSensitive = options.caseSensitive ?? !trie.isLegacy; - this.containsNoSuggestWords = options.noSuggest || false; - this._size = size || 0; - this.weightMap = options.weightMap || createWeightMapFromDictionaryInformation(options.dictionaryInformation); - this.ignoreCharactersRegExp = charsetToRegExp(this.options.dictionaryInformation?.ignore); - } - - public get size(): number { - if (!this._size) { - // walk the trie and get the approximate size. - const i = this.trie.iterate(); - let deeper = true; - let size = 0; - for (let r = i.next(); !r.done; r = i.next(deeper)) { - // count all nodes even though they are not words. - // because we are not going to all the leaves, this should give a good enough approximation. - size += 1; - deeper = r.value.text.length < 5; - } - this._size = size; - } - return this._size; - } - public has(word: string, hasOptions?: HasOptions): boolean { - const { useCompounds, ignoreCase } = this.resolveOptions(hasOptions); - const r = this._find(word, useCompounds, ignoreCase); - return !!r && !r.forbidden && !!r.found; - } - - public find(word: string, hasOptions?: HasOptions): FindResult | undefined { - const { useCompounds, ignoreCase } = this.resolveOptions(hasOptions); - const r = this._find(word, useCompounds, ignoreCase); - const { forbidden = this.isForbidden(word) } = r || {}; - if (!r && !forbidden) return undefined; - const { found = forbidden ? word : false } = r || {}; - const noSuggest = found !== false && this.containsNoSuggestWords; - return { found, forbidden, noSuggest }; - } - - private resolveOptions(hasOptions?: HasOptions): { - useCompounds: HasOptions['useCompounds'] | undefined; - ignoreCase: boolean; - } { - const { useCompounds = this.options.useCompounds, ignoreCase = true } = hasOptionToSearchOption(hasOptions); - return { useCompounds, ignoreCase }; - } - - private _find = memorizer( - (word: string, useCompounds: number | boolean | undefined, ignoreCase: boolean) => - this.findAnyForm(word, useCompounds, ignoreCase), - SpellingDictionaryFromTrie.cachedWordsLimit - ); - - private findAnyForm( - word: string, - useCompounds: number | boolean | undefined, - ignoreCase: boolean - ): FindAnyFormResult | undefined { - const outerForms = new Set([word]); - if (this.ignoreCharactersRegExp) { - outerForms.add(word.replace(this.ignoreCharactersRegExp, '')); - outerForms.add(word.normalize('NFD').replace(this.ignoreCharactersRegExp, '')); - outerForms.add(word.normalize('NFC').replace(this.ignoreCharactersRegExp, '')); - } - for (const form of outerForms) { - const r = this._findAnyForm(form, useCompounds, ignoreCase); - if (r) return r; - } - return undefined; - } - - private _findAnyForm( - word: string, - useCompounds: number | boolean | undefined, - ignoreCase: boolean - ): FindAnyFormResult | undefined { - const mWord = this.mapWord(word.normalize('NFC')); - const opts: FindWordOptions = { caseSensitive: !ignoreCase }; - const findResult = this.trie.findWord(mWord, opts); - if (findResult.found !== false) { - return findResult; - } - const forms = wordSearchForms(mWord, this.isDictionaryCaseSensitive, ignoreCase); - for (const w of forms) { - const findResult = this.trie.findWord(w, opts); - if (findResult.found !== false) { - return findResult; - } - } - if (useCompounds) { - opts.useLegacyWordCompounds = useCompounds; - for (const w of forms) { - const findResult = this.trie.findWord(w, opts); - if (findResult.found !== false) { - return findResult; - } - } - } - return undefined; - } - - public isNoSuggestWord(word: string, options?: HasOptions): boolean { - return this.containsNoSuggestWords ? this.has(word, options) : false; - } - - public isForbidden(word: string): boolean { - return this.trie.isForbiddenWord(word); - } - - public suggest( - word: string, - numSuggestions?: number, - compoundMethod?: CompoundWordsMethod, - numChanges?: number, - ignoreCase?: boolean - ): SuggestionResult[]; - public suggest(word: string, suggestOptions: SuggestOptions): SuggestionResult[]; - public suggest(...args: SuggestArgs): SuggestionResult[] { - const [word] = args; - const suggestOptions = suggestArgsToSuggestOptions(args); - return this._suggest(word, suggestOptions); - } - - private _suggest(word: string, suggestOptions: SuggestOptions): SuggestionResult[] { - const { - numSuggestions = getDefaultSettings(false).numSuggestions || defaultNumSuggestions, - numChanges, - includeTies, - ignoreCase, - timeout, - } = suggestOptions; - function filter(_word: string): boolean { - return true; - } - const collector = suggestionCollector( - word, - clean({ - numSuggestions, - filter, - changeLimit: numChanges, - includeTies, - ignoreCase, - timeout, - weightMap: this.weightMap, - }) - ); - this.genSuggestions(collector, suggestOptions); - return collector.suggestions.map((r) => ({ ...r, word: r.word })); - } - - public genSuggestions(collector: SuggestionCollector, suggestOptions: SuggestOptions): void { - if (this.options.noSuggest) return; - const _compoundMethod = - suggestOptions.compoundMethod ?? - (this.options.useCompounds ? CompoundWordsMethod.JOIN_WORDS : CompoundWordsMethod.NONE); - wordSuggestFormsArray(collector.word).forEach((w) => - this.trie.genSuggestions(impersonateCollector(collector, w), _compoundMethod) - ); - } - - public getErrors(): Error[] { - return []; - } -} - -type FindAnyFormResult = FindFullResult; - -export function createSpellingDictionaryTrie( - data: Iterable, - name: string, - source: string, - options: SpellingDictionaryOptions -): SpellingDictionary { - const trieNode = importTrie(data); - const trie = new Trie(trieNode); - return new SpellingDictionaryFromTrie(trie, name, options, source); -} diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryMethods.test.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryMethods.test.ts deleted file mode 100644 index 5434fe5db0b..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryMethods.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { impersonateCollector, suggestionCollector } from './SpellingDictionaryMethods'; - -describe('SpellingDictionaryMethods', () => { - test('impersonateCollector', () => { - const collector = suggestionCollector('hello', { numSuggestions: 1, changeLimit: 3, ignoreCase: true }); - const ic = impersonateCollector(collector, 'Hello'); - const suggestion = { word: 'hello', cost: 1 }; - ic.add(suggestion); - expect(ic.suggestions).toEqual([suggestion]); - expect(ic.maxCost).toBeGreaterThan(200); - expect(ic.maxNumSuggestions).toBe(1); - expect(ic.word).toBe('Hello'); - expect(collector.word).toBe('hello'); - }); -}); diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryMethods.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryMethods.ts deleted file mode 100644 index 9db6fc35a24..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionaryMethods.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { DictionaryInformation } from '@cspell/cspell-types'; -import { CompoundWordsMethod, mapDictionaryInformationToWeightMap, SuggestionResult, WeightMap } from 'cspell-trie-lib'; -import { clean } from 'cspell-trie-lib/dist/lib/trie-util'; -import { genSequence } from 'gensequence'; -import { isUpperCase, removeAccents, ucFirst } from '../../util/text'; -import { HasOptions, SearchOptions, SpellingDictionary, SuggestOptions } from './SpellingDictionary'; - -export { impersonateCollector, suggestionCollector } from 'cspell-trie-lib'; - -export type FilterSuggestionsPredicate = (word: SuggestionResult) => boolean; - -export type SuggestArgs = - | Parameters - | Parameters< - ( - word: string, - numSuggestions?: number, - compoundMethod?: CompoundWordsMethod, - numChanges?: number, - ignoreCase?: boolean - ) => SuggestionResult[] - >; - -export const defaultNumSuggestions = 10; - -function wordSearchFormsArray(word: string, isDictionaryCaseSensitive: boolean, ignoreCase: boolean): string[] { - return [...wordSearchForms(word, isDictionaryCaseSensitive, ignoreCase)]; -} - -export function wordSearchForms(word: string, isDictionaryCaseSensitive: boolean, ignoreCase: boolean): Set { - const forms = new Set(); - word = word.normalize('NFC'); - const wordLc = word.toLowerCase(); - if (ignoreCase) { - if (isDictionaryCaseSensitive) { - forms.add(wordLc); - } else { - forms.add(wordLc); - // Legacy remove any unbound accents - forms.add(wordLc.replace(/\p{M}/gu, '')); - } - } else { - if (isDictionaryCaseSensitive) { - forms.add(word); - forms.add(wordLc); - // HOUSE -> House, house - if (isUpperCase(word)) { - forms.add(ucFirst(wordLc)); - } - } else { - forms.add(wordLc); - // Legacy remove any unbound accents - forms.add(wordLc.replace(/\p{M}/gu, '')); - } - } - return forms; -} - -export function wordSuggestFormsArray(word: string): string[] { - return [...wordSuggestForms(word)]; -} - -export function wordSuggestForms(word: string): Set { - word = word.normalize('NFC'); - const forms = new Set([word]); - const wordLc = word.toLowerCase(); - forms.add(wordLc); - return forms; -} - -interface DictionaryWordForm { - w: string; // the word - p: string; // prefix to add -} -function* wordDictionaryForms(word: string, prefixNoCase: string): IterableIterator { - word = word.normalize('NFC'); - const wordLc = word.toLowerCase(); - const wordNa = removeAccents(word); - const wordLcNa = removeAccents(wordLc); - function wf(w: string, p = '') { - return { w, p }; - } - - const prefix = prefixNoCase; - yield wf(word); - yield wf(wordNa, prefix); - yield wf(wordLc, prefix); - yield wf(wordLcNa, prefix); -} - -export function wordDictionaryFormsCollector(prefixNoCase: string): (word: string) => Iterable { - const knownWords = new Set(); - - return (word: string) => { - return genSequence(wordDictionaryForms(word, prefixNoCase)) - .filter((w) => !knownWords.has(w.w)) - .map((w) => w.p + w.w) - .filter((w) => !knownWords.has(w)) - .map((w) => (knownWords.add(w), w)); - }; -} - -const DEFAULT_HAS_OPTIONS: HasOptions = Object.freeze({}); - -export function hasOptionToSearchOption(opt: HasOptions | undefined): SearchOptions { - return !opt ? DEFAULT_HAS_OPTIONS : opt; -} - -export function suggestArgsToSuggestOptions(args: SuggestArgs): SuggestOptions { - const [_word, options, compoundMethod, numChanges, ignoreCase] = args; - const suggestOptions: SuggestOptions = - typeof options === 'object' - ? options - : clean({ - numSuggestions: options, - compoundMethod, - numChanges, - ignoreCase, - includeTies: undefined, - timeout: undefined, - }); - return suggestOptions; -} -export function createWeightMapFromDictionaryInformation(di: undefined): undefined; -export function createWeightMapFromDictionaryInformation(di: DictionaryInformation): WeightMap; -export function createWeightMapFromDictionaryInformation(di: DictionaryInformation | undefined): WeightMap | undefined; -export function createWeightMapFromDictionaryInformation(di: DictionaryInformation | undefined): WeightMap | undefined { - return di ? mapDictionaryInformationToWeightMap(di) : undefined; -} - -export const __testMethods__ = { - wordSearchForms, - wordSearchFormsArray, - wordDictionaryForms, - wordDictionaryFormsCollector, -}; diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/charset.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/charset.ts deleted file mode 100644 index 8b4211d829e..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/charset.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { CharacterSet } from '@cspell/cspell-types'; - -export function charsetToRegExp(charset: CharacterSet | undefined): RegExp | undefined { - if (!charset) return undefined; - - try { - const reg = `[${charset.replace(/[\][\\]/g, '\\$&')}]`; - return new RegExp(reg, 'g'); - } catch (e) { - return undefined; - } -} diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary.test.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary.test.ts deleted file mode 100644 index cbb527c020b..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { SpellingDictionaryOptions } from './SpellingDictionary'; -import { DictionaryInformation } from '../..'; -import { mapDictDefToInternal } from '../../Settings/DictionarySettings'; -import { createFailedToLoadDictionary, createSpellingDictionary } from './createSpellingDictionary'; -import { SpellingDictionaryLoadError } from '../SpellingDictionaryError'; - -const di = mapDictDefToInternal; - -describe('Validate createSpellingDictionary', () => { - test('createFailedToLoadDictionary', () => { - const error = new Error('error'); - const loaderError = new SpellingDictionaryLoadError( - './missing.txt', - di({ name: 'failed dict', path: './missing.txt' }, __filename), - error, - 'Failed to load' - ); - const d = createFailedToLoadDictionary(loaderError); - expect(d).toBeTruthy(); - - expect(d.getErrors?.()).toEqual([loaderError]); - expect(d.suggest('error')).toEqual([]); - expect(d.mapWord('café')).toBe('café'); - expect(d.has('fun')).toBe(false); - expect(d.find('hello')).toBeUndefined(); - expect(d.isNoSuggestWord('hello', {})).toBe(false); - }); - - test('createSpellingDictionary', () => { - const words = ['one', 'two', 'three', 'left-right']; - const d = createSpellingDictionary(words, 'test create', __filename, opts()); - words.forEach((w) => expect(d.has(w)).toBe(true)); - }); - - test('createSpellingDictionary fa', () => { - // cspell:disable-next-line - const words = ['آئینهٔ', 'آبادهٔ', 'کلاه']; - expect(words).toEqual(words.map((w) => w.normalize('NFC'))); - const d = createSpellingDictionary(words, 'test create', __filename, opts()); - expect(d.has(words[0])).toBe(true); - words.forEach((w) => expect(d.has(w)).toBe(true)); - }); - - test('createSpellingDictionary fa legacy', () => { - // cspell:disable-next-line - const words = ['آئینهٔ', 'آبادهٔ', 'کلاه']; - expect(words).toEqual(words.map((w) => w.normalize('NFC'))); - const d = createSpellingDictionary( - words.map((w) => w.replace(/\p{M}/gu, '')), - 'test create', - __filename, - opts({ caseSensitive: false }) - ); - expect(d.has(words[0])).toBe(true); - words.forEach((w) => expect(d.has(w)).toBe(true)); - }); - - // cspell:ignore Geschäft Aujourd'hui - test('createSpellingDictionary accents', () => { - const words = ['Geschäft'.normalize('NFD'), 'café', 'book', "Aujourd'hui"]; - const d = createSpellingDictionary(words, 'test create', __filename, opts()); - expect(d.has(words[0])).toBe(true); - words.forEach((w) => expect(d.has(w)).toBe(true)); - words.map((w) => w.toLowerCase()).forEach((w) => expect(d.has(w)).toBe(true)); - expect(d.has(words[0].toLowerCase())).toBe(true); - expect(d.has(words[0].toLowerCase(), { ignoreCase: false })).toBe(false); - expect(d.suggest('geschaft', { ignoreCase: true }).map((r) => r.word)).toEqual([ - 'geschaft', - 'geschäft', - 'Geschäft', - ]); - expect(d.suggest('geschaft', { ignoreCase: false }).map((r) => r.word)).toEqual(['Geschäft']); - }); - - // cspell:ignore fone failor - test.each` - word | ignoreCase | expected - ${'Geschäft'} | ${false} | ${[c('Geschäft', 0)]} - ${'Geschaft'} | ${false} | ${[c('Geschäft', 1)]} - ${'fone'} | ${false} | ${[c('phone', 70), c('gone', 104)]} - ${'failor'} | ${false} | ${[c('failure', 70), c('sailor', 104), c('failed', 175), c('fail', 200)]} - `('createSpellingDictionary with dictionaryInformation "$word" "$ignoreCase"', ({ word, ignoreCase, expected }) => { - const words = sampleWords(); - const options = { ...opts(), dictionaryInformation: sampleDictionaryInformation({}) }; - const d = createSpellingDictionary(words, 'test create', __filename, options); - expect(d.suggest(word, { ignoreCase, numSuggestions: 4 })).toEqual(expected); - }); -}); - -function opts(opts: Partial = {}): SpellingDictionaryOptions { - return { - weightMap: undefined, - ...opts, - }; -} - -function c(word: string, cost: number) { - return { word, cost }; -} - -function sampleDictionaryInformation(di: DictionaryInformation = {}): DictionaryInformation { - const d: DictionaryInformation = { - suggestionEditCosts: [ - { - map: 'f(ph)(gh)|(ail)(ale)|(ur)(er)(ure)(or)', - replace: 70, - }, - { - map: 'aeiou', // cspell:ignore aeiou - replace: 75, - swap: 75, - }, - { - description: 'common vowel sounds.', - map: 'o(oh)(oo)|(oo)(ou)|(oa)(ou)', - replace: 65, - }, - ], - ...di, - }; - return d; -} - -function sampleWords() { - return [ - ...['Geschäft'.normalize('NFD'), 'café', 'book', "Aujourd'hui", 'cafe'], - ...['go', 'going', 'goes', 'gone'], - ...['phone', 'fall', 'phones', 'phoning', 'call', 'caller', 'called'], - ...['fail', 'fall', 'failed', 'failing', 'failure'], - ...['enough', 'though', 'through'], - ...['soup', 'soap', 'sooth', 'boot', 'boat'], - ...['sail', 'sailor', 'sailing', 'sails', 'sailed'], - ...['sale', 'sold', 'sales', 'selling'], - ...['tale', 'tail'], - ]; -} diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary.ts deleted file mode 100644 index f3fe7dc1a93..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { buildTrieFast, parseDictionaryLines } from 'cspell-trie-lib'; -import { deepEqual } from 'fast-equals'; -import { operators } from 'gensequence'; -import { IterableLike } from '../../util/IterableLike'; -import { AutoWeakCache, SimpleCache } from '../../util/simpleCache'; -import { SpellingDictionary, SpellingDictionaryOptions } from './SpellingDictionary'; -import { SpellingDictionaryLoadError } from '../SpellingDictionaryError'; -import { SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie'; -import { createWeightMapFromDictionaryInformation } from './SpellingDictionaryMethods'; - -const defaultOptions: SpellingDictionaryOptions = Object.freeze({ - weightMap: undefined, -}); - -type CreateSpellingDictionaryParams = Parameters; - -const cachedDictionaries = new AutoWeakCache( - _createSpellingDictionary, - 64 -); - -const maxSetSize = 3; -const cachedParamsByWordList = new SimpleCache>(64); - -export function createSpellingDictionary( - wordList: readonly string[] | IterableLike, - name: string, - source: string, - options: SpellingDictionaryOptions | undefined -): SpellingDictionary { - const params: CreateSpellingDictionaryParams = [wordList, name, source, options]; - - if (!Array.isArray(wordList)) { - return _createSpellingDictionary(params); - } - - const cached = cachedParamsByWordList.get(name) || new Set(); - - for (const cachedParams of cached) { - if (deepEqual(params, cachedParams)) { - return cachedDictionaries.get(cachedParams); - } - } - - if (cached.size > maxSetSize) cached.clear(); - cached.add(params); - cachedParamsByWordList.set(name, cached); - - return cachedDictionaries.get(params); -} - -function _createSpellingDictionary(params: CreateSpellingDictionaryParams): SpellingDictionary { - const [wordList, name, source, options] = params; - // console.log(`createSpellingDictionary ${name} ${source}`); - const parseOptions = { stripCaseAndAccents: options?.supportNonStrictSearches ?? true }; - const words = parseDictionaryLines(wordList, parseOptions); - const trie = buildTrieFast(words); - const opts = { ...(options || defaultOptions) }; - if (opts.weightMap === undefined && opts.dictionaryInformation) { - opts.weightMap = createWeightMapFromDictionaryInformation(opts.dictionaryInformation); - } - return new SpellingDictionaryFromTrie(trie, name, opts, source); -} - -export function createForbiddenWordsDictionary( - wordList: readonly string[], - name: string, - source: string, - options: SpellingDictionaryOptions | undefined -): SpellingDictionary { - // console.log(`createForbiddenWordsDictionary ${name} ${source}`); - const words = parseDictionaryLines(wordList.concat(wordList.map((a) => a.toLowerCase())), { - stripCaseAndAccents: !options?.noSuggest, - }); - const forbidWords = operators.map((w: string) => '!' + w)(words); - const trie = buildTrieFast(forbidWords); - return new SpellingDictionaryFromTrie(trie, name, options || defaultOptions, source); -} - -export function createFailedToLoadDictionary(error: SpellingDictionaryLoadError): SpellingDictionary { - const { options, uri: source } = error; - const errors = [error]; - return { - name: options.name, - source, - type: 'error', - containsNoSuggestWords: false, - has: () => false, - find: () => undefined, - isNoSuggestWord: () => false, - isForbidden: () => false, - suggest: () => [], - mapWord: (a) => a, - genSuggestions: () => { - return; - }, - size: 0, - options, - isDictionaryCaseSensitive: false, - getErrors: () => errors, - }; -} - -export function createIgnoreWordsDictionary( - wordList: readonly string[], - name: string, - source: string -): SpellingDictionary { - // console.log('createIgnoreWordsDictionary %o', wordList); - - return createSpellingDictionary(wordList, name, source, { - caseSensitive: true, - noSuggest: true, - weightMap: undefined, - supportNonStrictSearches: true, - }); -} diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/index.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/index.ts deleted file mode 100644 index 9bf1f0b9765..00000000000 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryLibOld/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -export * from './SpellingDictionary'; -import { createCollection } from './SpellingDictionaryCollection'; - -import { - createFailedToLoadDictionary, - createForbiddenWordsDictionary, - createIgnoreWordsDictionary, - createSpellingDictionary, -} from './createSpellingDictionary'; - -export const SpellingDictionaryLibOld = { - createCollection, - createFailedToLoadDictionary, - createForbiddenWordsDictionary, - createIgnoreWordsDictionary, - createSpellingDictionary, -} as const; diff --git a/packages/cspell-lib/src/textValidation/isWordValid.test.ts b/packages/cspell-lib/src/textValidation/isWordValid.test.ts index 9b485a3f1f4..5c1c2759068 100644 --- a/packages/cspell-lib/src/textValidation/isWordValid.test.ts +++ b/packages/cspell-lib/src/textValidation/isWordValid.test.ts @@ -1,6 +1,5 @@ -import { createCachingDictionary } from 'cspell-dictionary'; +import { createCachingDictionary, createSpellingDictionary } from 'cspell-dictionary'; import { createCollection, SpellingDictionaryOptions } from '../SpellingDictionary'; -import { createSpellingDictionary } from '../SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary'; import { __testing__ } from './isWordValid'; const { hasWordCheck } = __testing__; diff --git a/packages/cspell-lib/src/textValidation/textValidator.test.ts b/packages/cspell-lib/src/textValidation/textValidator.test.ts index 51d6fce17b6..03f885679b8 100644 --- a/packages/cspell-lib/src/textValidation/textValidator.test.ts +++ b/packages/cspell-lib/src/textValidation/textValidator.test.ts @@ -2,8 +2,12 @@ import { opConcatMap, opMap, pipeSync } from '@cspell/cspell-pipe/sync'; import { CSpellUserSettings, TextOffset } from '@cspell/cspell-types'; import { createCSpellSettingsInternal as csi } from '../Models/CSpellSettingsInternalDef'; import { finalizeSettings } from '../Settings'; -import { createCollection, getDictionaryInternal, SpellingDictionaryOptions } from '../SpellingDictionary'; -import { createSpellingDictionary } from '../SpellingDictionary/SpellingDictionaryLibOld/createSpellingDictionary'; +import { + createCollection, + createSpellingDictionary, + getDictionaryInternal, + SpellingDictionaryOptions, +} from '../SpellingDictionary'; import { FreqCounter } from '../util/FreqCounter'; import * as Text from '../util/text'; import { calcTextInclusionRanges, validateText, _testMethods } from './textValidator'; diff --git a/packages/cspell-lib/src/textValidation/textValidator.ts b/packages/cspell-lib/src/textValidation/textValidator.ts index fdc692c7923..21adb386268 100644 --- a/packages/cspell-lib/src/textValidation/textValidator.ts +++ b/packages/cspell-lib/src/textValidation/textValidator.ts @@ -1,6 +1,6 @@ import { opConcatMap, opFilter, opTake, pipe } from '@cspell/cspell-pipe/sync'; import { genSequence, Sequence } from 'gensequence'; -import { SpellingDictionary } from '../SpellingDictionary/SpellingDictionaryLibOld/SpellingDictionary'; +import { SpellingDictionary } from 'cspell-dictionary'; import * as Text from '../util/text'; import * as TextRange from '../util/TextRange'; import { lineValidatorFactory } from './lineValidatorFactory';