From f0ccda5abec9c236eb5387bcf6a6349f31cb81b7 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Fri, 20 Aug 2021 23:42:17 +0200 Subject: [PATCH] feat: Add support for `noSuggest` dictionaries. (#1554) * dev: add `noSuggestDictionaries` and `noSuggest` dictionary options. * dev: add some samples * test: Code coverage for Cache * dev: [trie-lib] add option to control adding normalized versions of words. * dev: correctly merge `noSuggestDictionaries` * dev: Integrate `isNoSuggestWord` and `isForbidden` into `SpellingDictionary` * dev: `@pattern` is not allowed on a non-string property * fix: Make sure the case setting is passed through to no suggest tests. --- .prettierignore | 3 +- cspell.schema.json | 75 ++++++++-- packages/cspell-lib/samples/.cspell.json | 2 +- .../.cspell/custom-dictionary.txt | 1 + .../configurations/.cspell/ignore-words-2.txt | 2 + .../configurations/.cspell/ignore-words.txt | 2 + .../samples/configurations/README.md | 7 + .../configurations/cspell-dictionaries.json | 33 +++++ .../samples/configurations/cspell.json | 6 + .../samples/yaml-config/cspell.yaml | 2 +- .../cspell-lib/src/Cache/cspell.cache.test.ts | 9 ++ packages/cspell-lib/src/Cache/cspell.cache.ts | 17 +-- packages/cspell-lib/src/Cache/index.test.ts | 7 + packages/cspell-lib/src/Cache/index.ts | 1 + .../src/Settings/CSpellSettingsServer.test.ts | 30 +++- .../src/Settings/CSpellSettingsServer.ts | 17 ++- .../src/Settings/DefaultSettings.ts | 2 +- .../DictionaryReferenceCollection.test.ts | 38 +++++ .../Settings/DictionaryReferenceCollection.ts | 66 +++++++++ .../src/Settings/DictionarySettings.test.ts | 17 +-- .../src/Settings/DictionarySettings.ts | 47 ++---- .../src/SpellingDictionary/Dictionaries.ts | 46 ++++-- .../SpellingDictionary/DictionaryLoader.ts | 13 +- .../SpellingDictionary.test.ts | 6 +- .../SpellingDictionary/SpellingDictionary.ts | 3 + .../SpellingDictionaryCollection.test.ts | 135 ++++++++++-------- .../SpellingDictionaryCollection.ts | 53 ++++--- .../SpellingDictionaryFromTrie.ts | 17 ++- .../createSpellingDictionary.test.ts | 6 +- .../createSpellingDictionary.ts | 22 ++- packages/cspell-lib/src/textValidator.test.ts | 66 ++++++--- packages/cspell-lib/src/textValidator.ts | 21 +-- packages/cspell-lib/src/validator.ts | 1 - .../src/lib/SimpleDictionaryParser.test.ts | 61 +++++++- .../src/lib/SimpleDictionaryParser.ts | 83 ++++++++--- packages/cspell-trie-lib/src/lib/constants.ts | 1 + packages/cspell-types/cspell.schema.json | 75 ++++++++-- .../src/settings/CSpellSettingsDef.ts | 86 ++++++++++- 38 files changed, 825 insertions(+), 254 deletions(-) create mode 100644 packages/cspell-lib/samples/configurations/.cspell/custom-dictionary.txt create mode 100644 packages/cspell-lib/samples/configurations/.cspell/ignore-words-2.txt create mode 100644 packages/cspell-lib/samples/configurations/.cspell/ignore-words.txt create mode 100644 packages/cspell-lib/samples/configurations/README.md create mode 100644 packages/cspell-lib/samples/configurations/cspell-dictionaries.json create mode 100644 packages/cspell-lib/samples/configurations/cspell.json create mode 100644 packages/cspell-lib/src/Cache/cspell.cache.test.ts create mode 100644 packages/cspell-lib/src/Cache/index.test.ts create mode 100644 packages/cspell-lib/src/Cache/index.ts create mode 100644 packages/cspell-lib/src/Settings/DictionaryReferenceCollection.test.ts create mode 100644 packages/cspell-lib/src/Settings/DictionaryReferenceCollection.ts diff --git a/.prettierignore b/.prettierignore index e1e208d8568..a22b4f48542 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,7 +1,8 @@ .vscode/ [sS]amples/ [tT]emp/ -**/dist/ +**/dist/** +**/node_modules/** **/package-lock.json CHANGELOG.md coverage diff --git a/cspell.schema.json b/cspell.schema.json index a4d551b7b71..a64cdd37c5f 100644 --- a/cspell.schema.json +++ b/cspell.schema.json @@ -45,7 +45,11 @@ }, "name": { "$ref": "#/definitions/DictionaryId", - "description": "The reference name of the dictionary, used with program language settings" + "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`" + }, + "noSuggest": { + "description": "Indicate that suggestions should not come from this dictionary. Words in this dictionary are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in this dictionary, it will be removed from the set of possible suggestions.", + "type": "boolean" }, "repMap": { "$ref": "#/definitions/ReplaceMap", @@ -76,7 +80,11 @@ }, "name": { "$ref": "#/definitions/DictionaryId", - "description": "The reference name of the dictionary, used with program language settings" + "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`" + }, + "noSuggest": { + "description": "Indicate that suggestions should not come from this dictionary. Words in this dictionary are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in this dictionary, it will be removed from the set of possible suggestions.", + "type": "boolean" }, "path": { "$ref": "#/definitions/CustomDictionaryPath", @@ -121,7 +129,11 @@ }, "name": { "$ref": "#/definitions/DictionaryId", - "description": "The reference name of the dictionary, used with program language settings" + "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`" + }, + "noSuggest": { + "description": "Indicate that suggestions should not come from this dictionary. Words in this dictionary are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in this dictionary, it will be removed from the set of possible suggestions.", + "type": "boolean" }, "path": { "$ref": "#/definitions/DictionaryPath", @@ -143,7 +155,13 @@ "type": "object" }, "DictionaryId": { - "description": "This matches the name in a dictionary definition", + "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`", + "pattern": "^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$", + "type": "string" + }, + "DictionaryNegRef": { + "description": "This a negative reference to a named dictionary.\n\nIt is used to exclude or include a dictionary by name.\n\nThe reference starts with 1 or more `!`.\n- `!` - Used to exclude the dictionary matching ``\n- `!!` - Used to re-include a dictionary matching `` Overrides `!`.\n- `!!!` - Used to exclude a dictionary matching `` Overrides `!!`.", + "pattern": "^(?=!+[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$", "type": "string" }, "DictionaryPath": { @@ -151,6 +169,22 @@ "pattern": "^.*\\.(?:txt|trie)(?:\\.gz)?$", "type": "string" }, + "DictionaryRef": { + "$ref": "#/definitions/DictionaryId", + "description": "This a reference to a named dictionary. It is expected to match the name of a dictionary.", + "pattern": "^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$" + }, + "DictionaryReference": { + "anyOf": [ + { + "$ref": "#/definitions/DictionaryRef" + }, + { + "$ref": "#/definitions/DictionaryNegRef" + } + ], + "description": "Reference to a dictionary by name. One of:\n- {@link DictionaryRef } \n- {@link DictionaryNegRef }" + }, "FsPath": { "description": "A File System Path", "type": "string" @@ -205,9 +239,9 @@ "type": "string" }, "dictionaries": { - "description": "Optional list of dictionaries to use.", + "description": "Optional list of dictionaries to use. Each entry should match the name of the dictionary. To remove a dictionary from the list add `!` before the name. i.e. `!typescript` will turn of the dictionary with the name `typescript`.", "items": { - "$ref": "#/definitions/DictionaryId" + "$ref": "#/definitions/DictionaryReference" }, "type": "array" }, @@ -289,6 +323,13 @@ "description": "Optional name of configuration", "type": "string" }, + "noSuggestDictionaries": { + "description": "Optional list of dictionaries that will not be used for suggestions. Words in these dictionaries are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in one of these dictionaries, it will be removed from the set of possible suggestions.", + "items": { + "$ref": "#/definitions/DictionaryReference" + }, + "type": "array" + }, "patterns": { "description": "Defines a list of patterns that can be used in ignoreRegExpList and includeRegExpList", "items": { @@ -331,9 +372,9 @@ "type": "string" }, "dictionaries": { - "description": "Optional list of dictionaries to use.", + "description": "Optional list of dictionaries to use. Each entry should match the name of the dictionary. To remove a dictionary from the list add `!` before the name. i.e. `!typescript` will turn of the dictionary with the name `typescript`.", "items": { - "$ref": "#/definitions/DictionaryId" + "$ref": "#/definitions/DictionaryReference" }, "type": "array" }, @@ -440,6 +481,13 @@ "description": "Optional name of configuration", "type": "string" }, + "noSuggestDictionaries": { + "description": "Optional list of dictionaries that will not be used for suggestions. Words in these dictionaries are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in one of these dictionaries, it will be removed from the set of possible suggestions.", + "items": { + "$ref": "#/definitions/DictionaryReference" + }, + "type": "array" + }, "numSuggestions": { "default": 10, "description": "Number of suggestions to make", @@ -629,9 +677,9 @@ "type": "string" }, "dictionaries": { - "description": "Optional list of dictionaries to use.", + "description": "Optional list of dictionaries to use. Each entry should match the name of the dictionary. To remove a dictionary from the list add `!` before the name. i.e. `!typescript` will turn of the dictionary with the name `typescript`.", "items": { - "$ref": "#/definitions/DictionaryId" + "$ref": "#/definitions/DictionaryReference" }, "type": "array" }, @@ -762,6 +810,13 @@ "description": "Prevents searching for local configuration when checking individual documents.", "type": "boolean" }, + "noSuggestDictionaries": { + "description": "Optional list of dictionaries that will not be used for suggestions. Words in these dictionaries are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in one of these dictionaries, it will be removed from the set of possible suggestions.", + "items": { + "$ref": "#/definitions/DictionaryReference" + }, + "type": "array" + }, "numSuggestions": { "default": 10, "description": "Number of suggestions to make", diff --git a/packages/cspell-lib/samples/.cspell.json b/packages/cspell-lib/samples/.cspell.json index da2c7dc5997..7ce71a8e3bf 100644 --- a/packages/cspell-lib/samples/.cspell.json +++ b/packages/cspell-lib/samples/.cspell.json @@ -47,7 +47,7 @@ "languageId": "c,cpp", // Language locale. i.e. en-US, de-AT, or ru. * will match all locales. // Multiple locales can be specified like: "en, en-US" to match both English and English US. - "local": "*", + "locale": "*", // By default the whole text of a file is included for spell checking // Adding patterns to the "includeRegExpList" to only include matching patterns "includeRegExpList": [ diff --git a/packages/cspell-lib/samples/configurations/.cspell/custom-dictionary.txt b/packages/cspell-lib/samples/configurations/.cspell/custom-dictionary.txt new file mode 100644 index 00000000000..00bb943f9af --- /dev/null +++ b/packages/cspell-lib/samples/configurations/.cspell/custom-dictionary.txt @@ -0,0 +1 @@ +# Custom Dictionary Words diff --git a/packages/cspell-lib/samples/configurations/.cspell/ignore-words-2.txt b/packages/cspell-lib/samples/configurations/.cspell/ignore-words-2.txt new file mode 100644 index 00000000000..8331fe7d99a --- /dev/null +++ b/packages/cspell-lib/samples/configurations/.cspell/ignore-words-2.txt @@ -0,0 +1,2 @@ +# List of words to be ignored. +nosugg2 diff --git a/packages/cspell-lib/samples/configurations/.cspell/ignore-words.txt b/packages/cspell-lib/samples/configurations/.cspell/ignore-words.txt new file mode 100644 index 00000000000..cd0dd36bf39 --- /dev/null +++ b/packages/cspell-lib/samples/configurations/.cspell/ignore-words.txt @@ -0,0 +1,2 @@ +# List of words to be ignored. +nosugg1 diff --git a/packages/cspell-lib/samples/configurations/README.md b/packages/cspell-lib/samples/configurations/README.md new file mode 100644 index 00000000000..a2f98ece0ea --- /dev/null +++ b/packages/cspell-lib/samples/configurations/README.md @@ -0,0 +1,7 @@ +# Sample CSpell Configurations + +- [cspell-dictionaries](./cspell-dictionaries.json) - example of loading custom dictionaries. + +nosugg1 + +nosugg2 diff --git a/packages/cspell-lib/samples/configurations/cspell-dictionaries.json b/packages/cspell-lib/samples/configurations/cspell-dictionaries.json new file mode 100644 index 00000000000..918f40c0188 --- /dev/null +++ b/packages/cspell-lib/samples/configurations/cspell-dictionaries.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json", + "version": "0.2", + "dictionaries": [ + "typescript", + "custom-dictionary", + "ignore-words" + ], + "noSuggestDictionaries": [ + "ignore-words-2" + ], + "import": [ + "@cspell/dict-python/cspell-ext.json" + ], + "dictionaryDefinitions": [ + { + "name": "custom-dictionary", + "path": "./.cspell/custom-dictionary.txt", + "addWords": true + }, + { + "name": "ignore-words", + "path": "./.cspell/ignore-words.txt", + "addWords": true, + "noSuggest": true + }, + { + "name": "ignore-words-2", + "path": "./.cspell/ignore-words-2.txt", + "addWords": true + } + ] +} diff --git a/packages/cspell-lib/samples/configurations/cspell.json b/packages/cspell-lib/samples/configurations/cspell.json new file mode 100644 index 00000000000..18ebd95322f --- /dev/null +++ b/packages/cspell-lib/samples/configurations/cspell.json @@ -0,0 +1,6 @@ +{ + "flagWords": [], + "import": [ + "./cspell-dictionaries.json" + ] +} diff --git a/packages/cspell-lib/samples/yaml-config/cspell.yaml b/packages/cspell-lib/samples/yaml-config/cspell.yaml index dc32dab6f78..461ed90fdf4 100644 --- a/packages/cspell-lib/samples/yaml-config/cspell.yaml +++ b/packages/cspell-lib/samples/yaml-config/cspell.yaml @@ -1,5 +1,5 @@ id: Yaml Example Config import: - - ./../cspell-imports.json + - ./../cspell-imports.json # Note this one does not exist on purpose. - ./../cspell-includes.json - ../.cspell.json diff --git a/packages/cspell-lib/src/Cache/cspell.cache.test.ts b/packages/cspell-lib/src/Cache/cspell.cache.test.ts new file mode 100644 index 00000000000..1bd9dfc4135 --- /dev/null +++ b/packages/cspell-lib/src/Cache/cspell.cache.test.ts @@ -0,0 +1,9 @@ +import { IssueCode } from './cspell.cache'; + +describe('Cache', () => { + test('IssueCode', () => { + const codes = [IssueCode.UnknownWord, IssueCode.ForbiddenWord, IssueCode.KnownIssue]; + const sum = codes.reduce((a, b) => a + b, 0); + expect(sum).toBe(IssueCode.ALL); + }); +}); diff --git a/packages/cspell-lib/src/Cache/cspell.cache.ts b/packages/cspell-lib/src/Cache/cspell.cache.ts index be19e7d9c79..1e15e7120c1 100644 --- a/packages/cspell-lib/src/Cache/cspell.cache.ts +++ b/packages/cspell-lib/src/Cache/cspell.cache.ts @@ -9,30 +9,31 @@ export interface CSpellCache { files: CachedFile[]; } -type Version = '0.1'; +export type Version = '0.1'; /** * Hash used. Starts with hash id. i.e. `sha1-` or `sha512-`. */ -type Hash = string; +export type Hash = string; -type UriRelPath = string; +export type UriRelPath = string; -enum IssueCode { +export enum IssueCode { UnknownWord = 1 << 0, ForbiddenWord = 1 << 1, KnownIssue = 1 << 2, + ALL = IssueCode.UnknownWord | IssueCode.ForbiddenWord | IssueCode.KnownIssue, } -interface CachedFile { +export interface CachedFile { hash: Hash; path: UriRelPath; issues: Issue[]; } -type Issue = IssueEntry | IssueLine; +export type Issue = IssueEntry | IssueLine; -interface IssueEntry { +export interface IssueEntry { line: number; character: number; code: IssueCode; @@ -40,7 +41,7 @@ interface IssueEntry { len: number; } -type IssueLine = [ +export type IssueLine = [ line: IssueEntry['line'], character: IssueEntry['character'], code: IssueEntry['code'], diff --git a/packages/cspell-lib/src/Cache/index.test.ts b/packages/cspell-lib/src/Cache/index.test.ts new file mode 100644 index 00000000000..c98d4d44d5d --- /dev/null +++ b/packages/cspell-lib/src/Cache/index.test.ts @@ -0,0 +1,7 @@ +import {} from './index'; + +describe('index', () => { + test('index', () => { + expect(true).toBe(true); + }); +}); diff --git a/packages/cspell-lib/src/Cache/index.ts b/packages/cspell-lib/src/Cache/index.ts new file mode 100644 index 00000000000..3fd016275b0 --- /dev/null +++ b/packages/cspell-lib/src/Cache/index.ts @@ -0,0 +1 @@ +export type { CSpellCache } from './cspell.cache'; diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts index 074c67eb940..c356bcf78f7 100644 --- a/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts +++ b/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts @@ -20,10 +20,12 @@ import { } from './CSpellSettingsServer'; import { getDefaultSettings, _defaultSettings } from './DefaultSettings'; import { CSpellSettingsWithSourceTrace, CSpellUserSettings, ImportFileRef } from '@cspell/cspell-types'; +import { logError, logWarning } from '../util/logger'; +import { mocked } from 'ts-jest/utils'; import * as path from 'path'; import { URI } from 'vscode-uri'; -const { normalizeSettings } = __testing__; +const { normalizeSettings, validateRawConfigVersion, validateRawConfigExports } = __testing__; const rootCspellLib = path.resolve(path.join(__dirname, '../..')); const samplesDir = path.resolve(rootCspellLib, 'samples'); @@ -31,6 +33,9 @@ const samplesSrc = path.join(samplesDir, 'src'); jest.mock('../util/logger'); +const mockedLogError = mocked(logError); +const mockedLogWarning = mocked(logWarning); + describe('Validate CSpellSettingsServer', () => { test('tests mergeSettings with conflicting "name"', () => { const left = { name: 'Left' }; @@ -415,6 +420,11 @@ describe('Validate Glob resolution', () => { }); describe('Validate search/load config files', () => { + beforeEach(() => { + mockedLogError.mockClear(); + mockedLogWarning.mockClear(); + }); + function importRefWithError(filename: string): ImportFileRefWithError { return { filename, @@ -537,6 +547,24 @@ describe('Validate search/load config files', () => { const result = await loadConfig(uriYarn2TestMedCspell, {}); expect(result.dictionaries).toEqual(['medical terms']); }); + + test.each` + config | mocked | expected + ${{ version: 'hello' }} | ${mockedLogError} | ${'Unsupported config file version: "hello"\n File: "filename"'} + ${{ version: '0.1' }} | ${mockedLogWarning} | ${'Legacy config file version found: "0.1", upgrade to "0.2"\n File: "filename"'} + ${{ version: '0.3' }} | ${mockedLogWarning} | ${'Newer config file version found: "0.3". Supported version is "0.2"\n File: "filename"'} + `('validateRawConfigVersion $config', ({ config, mocked, expected }) => { + validateRawConfigVersion(config, { filename: 'filename' }); + expect(mocked).toHaveBeenCalledWith(expected); + }); + + test('validateRawConfigExports', () => { + const d = { default: {}, name: '' }; + const c: CSpellUserSettings = d; + expect(() => validateRawConfigExports(c, { filename: 'filename' })).toThrowError( + 'Module `export default` is not supported.\n Use `module.exports =` instead.\n File: "filename"' + ); + }); }); function oc(v: Partial): T { diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts index de6d970605a..ddefbe2b60c 100644 --- a/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts +++ b/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts @@ -73,11 +73,15 @@ const searchPlaces = [ const cspellCosmiconfig: CosmicOptions & CosmicOptionsSync = { searchPlaces, loaders: { - '.json': (_filename: string, content: string) => json.parse(content), - '.jsonc': (_filename: string, content: string) => json.parse(content), + '.json': parseJson, + '.jsonc': parseJson, }, }; +function parseJson(_filename: string, content: string) { + return json.parse(content); +} + export const defaultConfigFilenames = Object.freeze(searchPlaces.concat()); const cspellConfigExplorer = cosmiconfig('cspell', cspellCosmiconfig); @@ -441,6 +445,7 @@ function merge(left: CSpellSettings, right: CSpellSettings): CSpellSettings { patterns: mergeListUnique(left.patterns, right.patterns), dictionaryDefinitions: mergeListUnique(left.dictionaryDefinitions, right.dictionaryDefinitions), dictionaries: mergeListUnique(left.dictionaries, right.dictionaries), + noSuggestDictionaries: mergeListUnique(left.noSuggestDictionaries, right.noSuggestDictionaries), languageSettings: mergeList( tagLanguageSettings(leftId, left.languageSettings), tagLanguageSettings(rightId, right.languageSettings) @@ -817,7 +822,7 @@ function validationMessage(msg: string, fileRef: ImportFileRef) { return msg + `\n File: "${fileRef.filename}"`; } -function validateRawConfigVersion(config: CSpellUserSettings, fileRef: ImportFileRef) { +function validateRawConfigVersion(config: CSpellUserSettings, fileRef: ImportFileRef): void { const { version } = config; if (version === undefined || supportedCSpellConfigVersions.includes(version)) return; @@ -829,12 +834,12 @@ function validateRawConfigVersion(config: CSpellUserSettings, fileRef: ImportFil const msg = version > currentSettingsFileVersion ? `Newer config file version found: "${version}". Supported version is "${currentSettingsFileVersion}"` - : `Legacy config file version found: "${version}". Upgrade to "${currentSettingsFileVersion}"`; + : `Legacy config file version found: "${version}", upgrade to "${currentSettingsFileVersion}"`; logWarning(validationMessage(msg, fileRef)); } -function validateRawConfigExports(config: CSpellUserSettings, fileRef: ImportFileRef) { +function validateRawConfigExports(config: CSpellUserSettings, fileRef: ImportFileRef): void { if ((<{ default: unknown }>config).default) { throw new ImportError( validationMessage('Module `export default` is not supported.\n Use `module.exports =` instead.', fileRef) @@ -849,4 +854,6 @@ function validateRawConfig(config: CSpellUserSettings, fileRef: ImportFileRef): export const __testing__ = { normalizeSettings, + validateRawConfigVersion, + validateRawConfigExports, }; diff --git a/packages/cspell-lib/src/Settings/DefaultSettings.ts b/packages/cspell-lib/src/Settings/DefaultSettings.ts index 53cc48eecf5..c639b21ad3d 100644 --- a/packages/cspell-lib/src/Settings/DefaultSettings.ts +++ b/packages/cspell-lib/src/Settings/DefaultSettings.ts @@ -129,7 +129,7 @@ const getSettings = (function () { if (!settings) { const jsonSettings = readSettings(defaultConfigFile); settings = mergeSettings(_defaultSettings, jsonSettings); - settings.name = jsonSettings.name || settings.name; + settings.name = jsonSettings.name; } return settings; }; diff --git a/packages/cspell-lib/src/Settings/DictionaryReferenceCollection.test.ts b/packages/cspell-lib/src/Settings/DictionaryReferenceCollection.test.ts new file mode 100644 index 00000000000..c346838c447 --- /dev/null +++ b/packages/cspell-lib/src/Settings/DictionaryReferenceCollection.test.ts @@ -0,0 +1,38 @@ +import { DictionaryReference } from '../../../cspell-types/dist'; +import { createDictionaryReferenceCollection } from './DictionaryReferenceCollection'; + +describe('DictionaryReferenceCollection', () => { + const dicts: DictionaryReference[] = ['typescript', '!!json', '!json', '!cpp', ' custom-dict ', '! python']; + + test.each` + name | expected + ${'typescript'} | ${true} + ${'json'} | ${true} + ${'cpp'} | ${false} + ${'unknown'} | ${undefined} + `('createDictionaryReferenceCollection.isEnabled $name', ({ name, expected }) => { + const collection = createDictionaryReferenceCollection(dicts); + expect(collection.isEnabled(name)).toBe(expected); + }); + + test.each` + name | expected + ${'typescript'} | ${false} + ${'json'} | ${false} + ${'cpp'} | ${true} + ${'unknown'} | ${undefined} + `('createDictionaryReferenceCollection.isBlocked $name', ({ name, expected }) => { + const collection = createDictionaryReferenceCollection(dicts); + expect(collection.isBlocked(name)).toBe(expected); + }); + + test('createDictionaryReferenceCollection.blocked', () => { + const collection = createDictionaryReferenceCollection(dicts); + expect(collection.blocked()).toEqual(['cpp', 'python']); + }); + + test('createDictionaryReferenceCollection.enabled', () => { + const collection = createDictionaryReferenceCollection(dicts); + expect(collection.enabled()).toEqual(['typescript', 'json', 'custom-dict']); + }); +}); diff --git a/packages/cspell-lib/src/Settings/DictionaryReferenceCollection.ts b/packages/cspell-lib/src/Settings/DictionaryReferenceCollection.ts new file mode 100644 index 00000000000..56b91974bd4 --- /dev/null +++ b/packages/cspell-lib/src/Settings/DictionaryReferenceCollection.ts @@ -0,0 +1,66 @@ +import { DictionaryId, DictionaryReference } from '@cspell/cspell-types'; + +export interface DictionaryReferenceCollection { + isEnabled(name: DictionaryId): boolean | undefined; + isBlocked(name: DictionaryId): boolean | undefined; + enabled(): DictionaryId[]; + blocked(): DictionaryId[]; + dictionaryIds: DictionaryId[]; +} + +export function createDictionaryReferenceCollection( + dictionaries: DictionaryReference[] +): DictionaryReferenceCollection { + return new _DictionaryReferenceCollection(dictionaries); +} + +class _DictionaryReferenceCollection implements DictionaryReferenceCollection { + readonly collection: Collection; + + constructor(readonly dictionaries: DictionaryReference[]) { + this.collection = collect(dictionaries); + } + + isEnabled(name: DictionaryId): boolean | undefined { + const entry = this.collection[name]; + return entry === undefined ? undefined : !!(entry & 0x1); + } + + isBlocked(name: DictionaryId): boolean | undefined { + const entry = this.collection[name]; + return entry === undefined ? undefined : !(entry & 0x1); + } + + enabled(): DictionaryId[] { + return this.dictionaryIds.filter((n) => this.isEnabled(n)); + } + + blocked(): DictionaryId[] { + return this.dictionaryIds.filter((n) => this.isBlocked(n)); + } + + get dictionaryIds(): DictionaryId[] { + return Object.keys(this.collection); + } +} + +type Collection = Record; + +function collect(dictionaries: DictionaryReference[]): Collection { + const refs = dictionaries.map(normalizeName).map(mapReference); + const col: Collection = {}; + for (const ref of refs) { + col[ref.name] = Math.max(ref.weight, col[ref.name] || 0); + } + return col; +} + +function normalizeName(entry: string): string { + return entry.normalize().trim(); +} + +function mapReference(ref: DictionaryReference): { name: string; weight: number } { + const name = ref.replace(/^!+/, ''); + const weight = ref.length - name.length + 1; + return { name: name.trim(), weight }; +} diff --git a/packages/cspell-lib/src/Settings/DictionarySettings.test.ts b/packages/cspell-lib/src/Settings/DictionarySettings.test.ts index 8b03de7c234..c21131cba3e 100644 --- a/packages/cspell-lib/src/Settings/DictionarySettings.test.ts +++ b/packages/cspell-lib/src/Settings/DictionarySettings.test.ts @@ -14,7 +14,7 @@ describe('Validate DictionarySettings', () => { ['php', 'wordsEn', 'unknown', 'en_us'], defaultSettings.dictionaryDefinitions! ); - const files = mapDefs.map((a) => a[1]).map((def) => def.name!); + const files = mapDefs.map((def) => def.name!); expect(mapDefs).toHaveLength(2); expect(files.filter((a) => a.includes('php'))).toHaveLength(1); expect(files.filter((a) => a.includes('wordsEn'))).toHaveLength(0); @@ -34,10 +34,7 @@ describe('Validate DictionarySettings', () => { ]; const expected = ['php', 'en_us'].sort(); const mapDefs = DictSettings.filterDictDefsToLoad(ids, defaultSettings.dictionaryDefinitions!); - const dicts = mapDefs - .map((a) => a[1]) - .map((def) => def.name!) - .sort(); + const dicts = mapDefs.map((def) => def.name!).sort(); expect(dicts).toEqual(expected); }); @@ -50,10 +47,7 @@ describe('Validate DictionarySettings', () => { const dictIds = ids.split(','); const expectedIds = expected.split(',').map((id) => id.trim()); const mapDefs = DictSettings.filterDictDefsToLoad(dictIds, defaultSettings.dictionaryDefinitions!); - const dicts = mapDefs - .map((a) => a[1]) - .map((def) => def.name!) - .sort(); + const dicts = mapDefs.map((def) => def.name!).sort(); expect(dicts).toEqual(expectedIds); }); @@ -61,10 +55,7 @@ describe('Validate DictionarySettings', () => { const defaultDicts = defaultSettings.dictionaryDefinitions!; const dictIds = defaultDicts.map((def) => def.name); const mapDefs = DictSettings.filterDictDefsToLoad(dictIds, defaultSettings.dictionaryDefinitions!); - const access = mapDefs - .map(([_, def]) => def) - .map((def) => def.path!) - .map((path) => fsp.access(path)); + const access = mapDefs.map((def) => def.path!).map((path) => fsp.access(path)); expect(mapDefs.length).toBeGreaterThan(0); return Promise.all(access); }); diff --git a/packages/cspell-lib/src/Settings/DictionarySettings.ts b/packages/cspell-lib/src/Settings/DictionarySettings.ts index 26dcd8703c7..56a7754bf2f 100644 --- a/packages/cspell-lib/src/Settings/DictionarySettings.ts +++ b/packages/cspell-lib/src/Settings/DictionarySettings.ts @@ -1,6 +1,7 @@ -import { DictionaryDefinition, DictionaryId, DictionaryDefinitionPreferred } from '@cspell/cspell-types'; +import { DictionaryDefinition, DictionaryDefinitionPreferred, DictionaryReference } from '@cspell/cspell-types'; import * as path from 'path'; import { resolveFile } from '../util/resolveFile'; +import { createDictionaryReferenceCollection } from './DictionaryReferenceCollection'; export interface DictionaryDefinitionWithSource extends DictionaryDefinitionPreferred { /** The path to the config file that contains this dictionary definition */ @@ -21,42 +22,22 @@ export type DefMapArrayItem = [string, DictionaryDefinitionPreferred]; * @param defs - dictionary definitions * @returns map from dictIds to definitions */ -export function filterDictDefsToLoad(dictIds: DictionaryId[], defs: DictionaryDefinition[]): DefMapArrayItem[] { - const negPrefixRegEx = /^!+/; - - // Collect the ids based upon the `!` depth. - const dictIdMap = dictIds - .map((id) => id.trim()) - .filter((id) => !!id) - .reduce((dictDepthMap, id) => { - const pfx = id.match(negPrefixRegEx); - const depth = pfx?.[0]?.length || 0; - const _dictSet = dictDepthMap.get(depth); - const dictSet = _dictSet || new Set(); - if (!_dictSet) { - dictDepthMap.set(depth, dictSet); - } - dictSet.add(id.slice(depth)); - return dictDepthMap; - }, new Map>()); - - const orderedSets = [...dictIdMap].sort((a, b) => a[0] - b[0]); - const dictIdSet = orderedSets.reduce((dictIdSet, [depth, ids]) => { - if (depth & 1) { - [...ids].forEach((id) => dictIdSet.delete(id)); - } else { - [...ids].forEach((id) => dictIdSet.add(id)); - } - return dictIdSet; - }, new Set()); +export function filterDictDefsToLoad( + dictRefIds: DictionaryReference[], + defs: DictionaryDefinition[] +): DictionaryDefinitionPreferred[] { + function isDefP(def: DictionaryDefinition): def is DictionaryDefinitionPreferred { + return !!def.path; + } - const activeDefs: DefMapArrayItem[] = defs + const col = createDictionaryReferenceCollection(dictRefIds); + const dictIdSet = new Set(col.enabled()); + const allActiveDefs = defs .filter(({ name }) => dictIdSet.has(name)) .map((def) => ({ ...def, path: getFullPathName(def) })) // Remove any empty paths. - .filter((def) => !!def.path) - .map((def) => [def.name, def] as DefMapArrayItem); - return [...new Map(activeDefs)]; + .filter(isDefP); + return [...new Map(allActiveDefs.map((d) => [d.name, d])).values()]; } function getFullPathName(def: DictionaryDefinition) { diff --git a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts index 94041bb7348..e247130f549 100644 --- a/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts +++ b/packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts @@ -1,16 +1,19 @@ -import { DictionaryDefinition, DictionaryId, CSpellUserSettings } from '@cspell/cspell-types'; +import { CSpellUserSettings, DictionaryDefinition, DictionaryReference } from '@cspell/cspell-types'; +import { createDictionaryReferenceCollection } from '../Settings/DictionaryReferenceCollection'; import { filterDictDefsToLoad } from '../Settings/DictionarySettings'; +import { createForbiddenWordsDictionary, createSpellingDictionary } from './createSpellingDictionary'; import { loadDictionary, refreshCacheEntries } from './DictionaryLoader'; -import { createSpellingDictionary } from './createSpellingDictionary'; +import { SpellingDictionaryCollection } from './index'; import { SpellingDictionary } from './SpellingDictionary'; import { createCollectionP } from './SpellingDictionaryCollection'; -import { SpellingDictionaryCollection } from './index'; -export function loadDictionaries(dictIds: DictionaryId[], defs: DictionaryDefinition[]): Promise[] { +export function loadDictionaries( + dictIds: DictionaryReference[], + defs: DictionaryDefinition[] +): Promise[] { const defsToLoad = filterDictDefsToLoad(dictIds, defs); return defsToLoad - .map((e) => e[1]) .map((def) => loadDictionary(def.path, def)) .map((p) => p.catch(() => undefined)) .filter((p) => !!p) @@ -22,12 +25,33 @@ export function refreshDictionaryCache(maxAge?: number): Promise { } export function getDictionary(settings: CSpellUserSettings): Promise { - const { words = [], userWords = [], dictionaries = [], dictionaryDefinitions = [], flagWords = [] } = settings; - const spellDictionaries = loadDictionaries(dictionaries, dictionaryDefinitions); - const settingsDictionary = createSpellingDictionary(words.concat(userWords), 'user_words', 'From Settings'); + const { + words = [], + userWords = [], + dictionaries = [], + dictionaryDefinitions = [], + noSuggestDictionaries = [], + flagWords = [], + ignoreWords = [], + } = settings; + const colNoSug = createDictionaryReferenceCollection(noSuggestDictionaries); + const colDicts = createDictionaryReferenceCollection(dictionaries.concat(colNoSug.enabled())); + const modDefs = dictionaryDefinitions.map((def) => { + const enabled = colNoSug.isEnabled(def.name); + if (enabled === undefined) return def; + return { ...def, noSuggest: enabled }; + }); + const spellDictionaries = loadDictionaries(colDicts.enabled(), modDefs); + const settingsDictionary = createSpellingDictionary(words.concat(userWords), 'user_words', 'From Settings', { + caseSensitive: true, + }); + const ignoreWordsDictionary = createSpellingDictionary(ignoreWords, 'ignore_words', 'From Settings', { + caseSensitive: true, + noSuggest: true, + }); + const flagWordsDictionary = createForbiddenWordsDictionary(flagWords, 'flag_words', 'From Settings', {}); return createCollectionP( - [...spellDictionaries, Promise.resolve(settingsDictionary)], - 'dictionary collection', - flagWords + [...spellDictionaries, settingsDictionary, ignoreWordsDictionary, flagWordsDictionary], + 'dictionary collection' ); } diff --git a/packages/cspell-lib/src/SpellingDictionary/DictionaryLoader.ts b/packages/cspell-lib/src/SpellingDictionary/DictionaryLoader.ts index 1b43f65ce2a..2fa7f66bc66 100644 --- a/packages/cspell-lib/src/SpellingDictionary/DictionaryLoader.ts +++ b/packages/cspell-lib/src/SpellingDictionary/DictionaryLoader.ts @@ -39,7 +39,7 @@ export interface Loaders { const dictionaryCache = new Map(); export function loadDictionary(uri: string, options: DictionaryDefinitionPreferred): Promise { - const key = calcKey(uri); + const key = calcKey(uri, options); const entry = dictionaryCache.get(key); if (entry) { return entry.dictionary; @@ -49,9 +49,14 @@ export function loadDictionary(uri: string, options: DictionaryDefinitionPreferr return loadedEntry.dictionary; } -function calcKey(uri: string) { +const importantOptionKeys: (keyof DictionaryDefinitionPreferred)[] = ['noSuggest', 'useCompounds']; + +function calcKey(uri: string, options: DictionaryDefinitionPreferred) { const loaderType = determineType(uri); - return [uri, loaderType].join('|'); + const optValues = importantOptionKeys.map((k) => options[k]?.toString() || ''); + const parts = [uri, loaderType].concat(optValues); + + return parts.join('|'); } /** @@ -70,7 +75,7 @@ async function refreshEntry(entry: CacheEntry, maxAge: number, now: number): Pro const pStat = stat(entry.uri).catch((e) => e as Error); const [state, oldState] = await Promise.all([pStat, entry.state]); if (entry.ts === now && !isEqual(state, oldState)) { - dictionaryCache.set(calcKey(entry.uri), loadEntry(entry.uri, entry.options)); + dictionaryCache.set(calcKey(entry.uri, entry.options), loadEntry(entry.uri, entry.options)); } } } diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.test.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.test.ts index f4ad678f342..0094044518b 100644 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.test.ts +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.test.ts @@ -9,7 +9,7 @@ describe('Verify building Dictionary', () => { test('build from word list', async () => { const words = ['apple', 'ape', 'able', 'apple', 'banana', 'orange', 'pear', 'aim', 'approach', 'bear']; - const dict = await createSpellingDictionary(words, 'words', 'test'); + const dict = await createSpellingDictionary(words, 'words', 'test', {}); expect(dict.name).toBe('words'); expect(dict.has('apple')).toBe(true); const suggestions = dict.suggest('aple').map(({ word }) => word); @@ -89,7 +89,7 @@ describe('Verify building Dictionary', () => { 'tattles', ]; const trie = Trie.create(words); - const dict = new SpellingDictionaryFromTrie(trie, 'trie'); + const dict = new SpellingDictionaryFromTrie(trie, 'trie', {}); // cspell:ignore cattles const results = dict.suggest('Cattles'); const suggestions = results.map(({ word }) => word); @@ -101,7 +101,7 @@ describe('Verify building Dictionary', () => { // eslint-disable-next-line no-sparse-arrays const words = ['apple', 'ape', 'able', , 'apple', 'banana', 'orange', 5, 'pear', 'aim', 'approach', 'bear']; - const dict = await createSpellingDictionary(words as string[], 'words', 'test'); + const dict = await createSpellingDictionary(words as string[], 'words', 'test', {}); expect(dict.name).toBe('words'); // expect(dict).toBeInstanceOf(SpellingDictionaryFromTrie); expect(dict.has('apple')).toBe(true); diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts index b0a2c448bf6..4a5a67c4d47 100644 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts @@ -28,16 +28,19 @@ export interface SpellingDictionaryOptions { repMap?: ReplaceMap; useCompounds?: boolean; caseSensitive?: boolean; + noSuggest?: boolean; } export interface SpellingDictionary { readonly name: string; readonly type: string; readonly source: string; + readonly containsNoSuggestWords: boolean; has(word: string, useCompounds: boolean): boolean; has(word: string, options: HasOptions): boolean; has(word: string, options?: HasOptions): boolean; isForbidden(word: string): boolean; + isNoSuggestWord(word: string, options: HasOptions): boolean; suggest( word: string, numSuggestions?: number, diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.test.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.test.ts index 5c21ec9542e..7758226e9c9 100644 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.test.ts +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.test.ts @@ -1,7 +1,11 @@ import * as Trie from 'cspell-trie-lib'; import { SpellingDictionaryCollection, createCollectionP, createCollection } from './SpellingDictionaryCollection'; import { CompoundWordsMethod } from './SpellingDictionaryMethods'; -import { createFailedToLoadDictionary, createSpellingDictionary } from './createSpellingDictionary'; +import { + createFailedToLoadDictionary, + createForbiddenWordsDictionary, + createSpellingDictionary, +} from './createSpellingDictionary'; import { SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie'; import { SpellingDictionaryLoadError } from './SpellingDictionaryError'; @@ -26,13 +30,14 @@ describe('Verify using multiple dictionaries', () => { const wordsF = ['!pink*', '+berry', '+bug', '!stinkbug']; test('checks for existence', async () => { const dicts = await Promise.all([ - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), - createSpellingDictionary(wordsD, 'wordsD', 'test'), + createSpellingDictionary(wordsA, 'wordsA', 'test', {}), + createSpellingDictionary(wordsB, 'wordsB', 'test', {}), + createSpellingDictionary(wordsC, 'wordsC', 'test', {}), + createSpellingDictionary(wordsD, 'wordsD', 'test', {}), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), ]); - const dictCollection = new SpellingDictionaryCollection(dicts, 'test', ['Avocado']); + const dictCollection = new SpellingDictionaryCollection(dicts, 'test'); expect(dictCollection.has('mango')).toBe(true); expect(dictCollection.has('tree')).toBe(false); expect(dictCollection.has('avocado')).toBe(false); @@ -44,19 +49,19 @@ describe('Verify using multiple dictionaries', () => { }); test('checks mapWord is identity', async () => { - const dicts = await Promise.all([createSpellingDictionary(wordsA, 'wordsA', 'test')]); + const dicts = await Promise.all([createSpellingDictionary(wordsA, 'wordsA', 'test', {})]); - const dictCollection = new SpellingDictionaryCollection(dicts, 'test', []); + const dictCollection = new SpellingDictionaryCollection(dicts, 'test'); expect(dictCollection.mapWord('Hello')).toBe('Hello'); }); test('checks for suggestions', async () => { - const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA'); + const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA', {}); const dicts = await Promise.all([ trie, - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), + createSpellingDictionary(wordsB, 'wordsB', 'test', {}), + createSpellingDictionary(wordsA, 'wordsA', 'test', {}), + createSpellingDictionary(wordsC, 'wordsC', 'test', {}), createFailedToLoadDictionary( new SpellingDictionaryLoadError( './missing.txt', @@ -65,9 +70,10 @@ describe('Verify using multiple dictionaries', () => { 'failed to load' ) ), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), ]); - const dictCollection = createCollection(dicts, 'test', ['Avocado']); + const dictCollection = createCollection(dicts, 'test'); expect(dictCollection.getErrors?.()).toHaveLength(1); const sugsForTango = dictCollection.suggest('tango', 10); expect(sugsForTango).toHaveLength(1); @@ -78,17 +84,18 @@ describe('Verify using multiple dictionaries', () => { test('checks for compound suggestions', async () => { // Add "wordsA" twice, once as a compound dictionary and once as a normal dictionary. - const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA'); + const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA', {}); trie.options.useCompounds = true; const dicts = await Promise.all([ trie, - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), ]); // cspell:ignore appletango applemango - const dictCollection = createCollection(dicts, 'test', ['Avocado']); + const dictCollection = createCollection(dicts, 'test'); const sugResult = dictCollection.suggest('appletango', 10, CompoundWordsMethod.SEPARATE_WORDS); const sugs = sugResult.map((a) => a.word); expect(sugs).toHaveLength(10); @@ -97,16 +104,17 @@ describe('Verify using multiple dictionaries', () => { }); test('checks for compound suggestions with numbChanges', async () => { - const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA'); + const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA', {}); const dicts = await Promise.all([ trie, - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), ]); // cspell:ignore appletango applemango - const dictCollection = createCollection(dicts, 'test', ['Avocado']); + const dictCollection = createCollection(dicts, 'test'); const sugResult = dictCollection.suggest('appletango', 10, CompoundWordsMethod.SEPARATE_WORDS, 2); const sugs = sugResult.map((a) => a.word); expect(sugs).toHaveLength(1); @@ -117,20 +125,21 @@ describe('Verify using multiple dictionaries', () => { test.each` word | expected ${'redberry'} | ${true} - ${'pink'} | ${true} + ${'pink'} | ${false} ${'bug'} | ${true} ${'blackberry'} | ${true} ${'pinkbug'} | ${true} `('checks has word: "$word"', ({ word, expected }) => { const dicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), - createSpellingDictionary(wordsD, 'wordsD', 'test'), - createSpellingDictionary(wordsF, 'wordsF', 'test'), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), + createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), + createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), ]; - const dictCollection = createCollection(dicts, 'test', ['Avocado']); + const dictCollection = createCollection(dicts, 'test'); expect(dictCollection.has(word)).toEqual(expected); }); @@ -146,14 +155,15 @@ describe('Verify using multiple dictionaries', () => { ${'pinkbug'} | ${false} `('checks forbid word: "$word"', ({ word, expected }) => { const dicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), - createSpellingDictionary(wordsD, 'wordsD', 'test'), - createSpellingDictionary(wordsF, 'wordsF', 'test'), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), + createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), + createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), ]; - const dictCollection = createCollection(dicts, 'test', ['Avocado']); + const dictCollection = createCollection(dicts, 'test'); expect(dictCollection.isForbidden(word)).toEqual(expected); }); @@ -170,38 +180,40 @@ describe('Verify using multiple dictionaries', () => { ${'stinkbug'} | ${[sr('stink bug', 103), sr('pinkbug', 198)]} `('checks suggestions word: "$word"', ({ word, expected }) => { const dicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), - createSpellingDictionary(wordsD, 'wordsD', 'test'), - createSpellingDictionary(wordsF, 'wordsF', 'test'), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), + createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), + createSpellingDictionary(wordsF, 'wordsF', 'test', undefined), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), ]; - const dictCollection = createCollection(dicts, 'test', ['Avocado']); + const dictCollection = createCollection(dicts, 'test'); expect(dictCollection.suggest(word, 2)).toEqual(expected); }); test('checks for suggestions with flagged words', async () => { const dicts = await Promise.all([ - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), ]); - const dictCollection = createCollection(dicts, 'test', ['Avocado']); + const dictCollection = createCollection(dicts, 'test'); const sugs = dictCollection.suggest('avocado', 10); expect(sugs.map((r) => r.word)).not.toContain('avocado'); }); test('checks for suggestions from mixed sources', async () => { const dicts = await Promise.all([ - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), ]); - const dictCollection = new SpellingDictionaryCollection(dicts, 'test', []); + const dictCollection = new SpellingDictionaryCollection(dicts, 'test'); expect(dictCollection.has('mango')).toBe(true); expect(dictCollection.has('lion')).toBe(true); expect(dictCollection.has('ant')).toBe(true); @@ -221,12 +233,12 @@ describe('Verify using multiple dictionaries', () => { test('creates using createCollectionP', async () => { const dicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), ]; - const dictCollection = await createCollectionP(dicts, 'test', []); + const dictCollection = await createCollectionP(dicts, 'test'); expect(dictCollection.has('mango')).toBe(true); expect(dictCollection.has('tree')).toBe(false); const sugs = dictCollection.suggest('mangos', 4); @@ -257,14 +269,15 @@ describe('Validate looking up words', () => { const cities = ['Seattle', 'Berlin', 'Amsterdam', 'Rome', 'London', 'Mumbai', 'Tokyo']; const testDicts = [ - createSpellingDictionary(wordsA, 'wordsA', 'test'), - createSpellingDictionary(wordsB, 'wordsB', 'test'), - createSpellingDictionary(wordsC, 'wordsC', 'test'), - createSpellingDictionary(wordsD, 'wordsD', 'test'), - createSpellingDictionary(cities, 'cities', 'test'), + createSpellingDictionary(wordsA, 'wordsA', 'test', undefined), + createSpellingDictionary(wordsB, 'wordsB', 'test', undefined), + createSpellingDictionary(wordsC, 'wordsC', 'test', undefined), + createSpellingDictionary(wordsD, 'wordsD', 'test', undefined), + createSpellingDictionary(cities, 'cities', 'test', undefined), + createForbiddenWordsDictionary(['Avocado'], 'flag_words', 'test', undefined), ]; - const testDictCollection = new SpellingDictionaryCollection(testDicts, 'test', ['Avocado']); + const testDictCollection = new SpellingDictionaryCollection(testDicts, 'test'); interface HasWordTest { word: string; diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.ts index c5b3c4275a6..7c42567bbc8 100644 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.ts +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryCollection.ts @@ -18,7 +18,7 @@ import { import { CASE_INSENSITIVE_PREFIX } from 'cspell-trie-lib'; import { genSequence } from 'gensequence'; import { getDefaultSettings } from '../Settings'; -import { memorizer } from '../util/Memorizer'; +import { memorizer, memorizerKeyBy } from '../util/Memorizer'; import { SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie'; function identityString(w: string): string { @@ -28,25 +28,29 @@ function identityString(w: string): string { export class SpellingDictionaryCollection implements SpellingDictionary { readonly options: SpellingDictionaryOptions = {}; readonly mapWord = identityString; - readonly wordsToFlag: Set; readonly type = 'SpellingDictionaryCollection'; readonly source: string; readonly isDictionaryCaseSensitive: boolean; + readonly containsNoSuggestWords: boolean; - constructor(readonly dictionaries: SpellingDictionary[], readonly name: string, wordsToFlag: string[]) { + constructor(readonly dictionaries: SpellingDictionary[], readonly name: string) { this.dictionaries = this.dictionaries.sort((a, b) => b.size - a.size); - this.wordsToFlag = new Set(wordsToFlag.map((w) => w.toLowerCase())); this.source = dictionaries.map((d) => d.name).join(', '); this.isDictionaryCaseSensitive = this.dictionaries.reduce((a, b) => a || b.isDictionaryCaseSensitive, false); + this.containsNoSuggestWords = this.dictionaries.reduce((a, b) => a || b.containsNoSuggestWords, false); } public has(word: string, hasOptions?: HasOptions): boolean { const options = hasOptionToSearchOption(hasOptions); - return !this.wordsToFlag.has(word.toLowerCase()) && !!isWordInAnyDictionary(this.dictionaries, word, options); + return !!isWordInAnyDictionary(this.dictionaries, word, options) && !this.isForbidden(word); + } + + public isNoSuggestWord(word: string, options?: HasOptions): boolean { + return this._isNoSuggestWord(word, options); } public isForbidden(word: string): boolean { - return this.wordsToFlag.has(word.toLowerCase()) || !!this._isForbiddenInDict(word); + return !!this._isForbiddenInDict(word) && !this.isNoSuggestWord(word); } public suggest( @@ -75,9 +79,9 @@ export class SpellingDictionaryCollection implements SpellingDictionary { const prefixNoCase = CASE_INSENSITIVE_PREFIX; const filter = (word: string, _cost: number) => { return ( - !this.wordsToFlag.has(word.toLowerCase()) && (ignoreCase || word[0] !== prefixNoCase) && - !this.isForbidden(word) + !this.isForbidden(word) && + !this.isNoSuggestWord(word, suggestOptions) ); }; const collector = suggestionCollector(word, { @@ -110,14 +114,22 @@ export class SpellingDictionaryCollection implements SpellingDictionary { (word: string) => isWordForbiddenInAnyDictionary(this.dictionaries, word), SpellingDictionaryFromTrie.cachedWordsLimit ); + + private _isNoSuggestWord = memorizerKeyBy( + (word: string, options?: HasOptions) => { + if (!this.containsNoSuggestWords) return false; + return !!isNoSuggestWordInAnyDictionary(this.dictionaries, word, options || false); + }, + (word: string, options?: HasOptions) => { + const opts = hasOptionToSearchOption(options); + return [word, opts.useCompounds, opts.ignoreCase].join(); + }, + SpellingDictionaryFromTrie.cachedWordsLimit + ); } -export function createCollection( - dictionaries: SpellingDictionary[], - name: string, - wordsToFlag: string[] = [] -): SpellingDictionaryCollection { - return new SpellingDictionaryCollection(dictionaries, name, wordsToFlag); +export function createCollection(dictionaries: SpellingDictionary[], name: string): SpellingDictionaryCollection { + return new SpellingDictionaryCollection(dictionaries, name); } function isWordInAnyDictionary( @@ -128,16 +140,23 @@ function isWordInAnyDictionary( return genSequence(dicts).first((dict) => dict.has(word, options)); } +function isNoSuggestWordInAnyDictionary( + dicts: SpellingDictionary[], + word: string, + options: HasOptions +): SpellingDictionary | undefined { + return genSequence(dicts).first((dict) => dict.isNoSuggestWord(word, options)); +} + function isWordForbiddenInAnyDictionary(dicts: SpellingDictionary[], word: string): SpellingDictionary | undefined { return genSequence(dicts).first((dict) => dict.isForbidden(word)); } export function createCollectionP( dicts: (Promise | SpellingDictionary)[], - name: string, - wordsToFlag: string[] + name: string ): Promise { - return Promise.all(dicts).then((dicts) => new SpellingDictionaryCollection(dicts, name, wordsToFlag)); + return Promise.all(dicts).then((dicts) => new SpellingDictionaryCollection(dicts, name)); } export const __testing__ = { diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts index 857df233d25..1870756065e 100644 --- a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts +++ b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts @@ -27,17 +27,21 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { readonly mapWord: (word: string) => string; readonly type = 'SpellingDictionaryFromTrie'; readonly isDictionaryCaseSensitive: boolean; + readonly containsNoSuggestWords: boolean; + constructor( readonly trie: Trie, readonly name: string, - readonly options: SpellingDictionaryOptions = {}, + readonly options: SpellingDictionaryOptions, readonly source = 'from trie', size?: number ) { this.mapWord = createMapper(options.repMap || []); this.isDictionaryCaseSensitive = options.caseSensitive ?? !trie.isLegacy; + this.containsNoSuggestWords = options.noSuggest || false; this._size = size || 0; } + public get size(): number { if (!this._size) { // walk the trie and get the approximate size. @@ -60,11 +64,13 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { const { ignoreCase = true } = searchOptions; return this._has(word, useCompounds, ignoreCase); } + private _has = memorizer( (word: string, useCompounds: number | boolean | undefined, ignoreCase: boolean) => this.hasAnyForm(word, useCompounds, ignoreCase), SpellingDictionaryFromTrie.cachedWordsLimit ); + private hasAnyForm(word: string, useCompounds: number | boolean | undefined, ignoreCase: boolean) { const mWord = this.mapWord(word.normalize('NFC')); if (this.trie.hasWord(mWord, true)) { @@ -86,6 +92,10 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { return false; } + public isNoSuggestWord(word: string, options?: HasOptions): boolean { + return this.containsNoSuggestWords ? this.has(word, options) : false; + } + public isForbidden(word: string): boolean { return this.trie.isForbiddenWord(word); } @@ -103,6 +113,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { const suggestOptions = suggestArgsToSuggestOptions(args); return this._suggest(word, suggestOptions); } + private _suggest(word: string, suggestOptions: SuggestOptions): SuggestionResult[] { const { numSuggestions = getDefaultSettings().numSuggestions || defaultNumSuggestions, @@ -122,7 +133,9 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { this.genSuggestions(collector, suggestOptions); return collector.suggestions.map((r) => ({ ...r, word: r.word })); } + public genSuggestions(collector: SuggestionCollector, suggestOptions: SuggestOptions): void { + if (this.options.noSuggest) return; const { compoundMethod = CompoundWordsMethod.SEPARATE_WORDS } = suggestOptions; const _compoundMethod = this.options.useCompounds ? CompoundWordsMethod.JOIN_WORDS : compoundMethod; wordSuggestFormsArray(collector.word).forEach((w) => @@ -139,7 +152,7 @@ export async function createSpellingDictionaryTrie( data: Iterable, name: string, source: string, - options?: SpellingDictionaryOptions + options: SpellingDictionaryOptions ): Promise { const trieNode = importTrie(data); const trie = new Trie(trieNode); diff --git a/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.test.ts b/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.test.ts index 0255472ab30..c24dc3fe5bc 100644 --- a/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.test.ts +++ b/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.test.ts @@ -21,7 +21,7 @@ describe('Validate createSpellingDictionary', () => { test('createSpellingDictionary', () => { const words = ['one', 'two', 'three', 'left-right']; - const d = createSpellingDictionary(words, 'test create', __filename); + const d = createSpellingDictionary(words, 'test create', __filename, {}); words.forEach((w) => expect(d.has(w)).toBe(true)); }); @@ -29,7 +29,7 @@ describe('Validate createSpellingDictionary', () => { // cspell:disable-next-line const words = ['آئینهٔ', 'آبادهٔ', 'کلاه']; expect(words).toEqual(words.map((w) => w.normalize('NFC'))); - const d = createSpellingDictionary(words, 'test create', __filename); + const d = createSpellingDictionary(words, 'test create', __filename, {}); expect(d.has(words[0])).toBe(true); words.forEach((w) => expect(d.has(w)).toBe(true)); }); @@ -51,7 +51,7 @@ describe('Validate createSpellingDictionary', () => { // cspell:ignore Geschäft Aujourd'hui test('createSpellingDictionary accents', () => { const words = ['Geschäft'.normalize('NFD'), 'café', 'book', "Aujourd'hui"]; - const d = createSpellingDictionary(words, 'test create', __filename); + const d = createSpellingDictionary(words, 'test create', __filename, {}); expect(d.has(words[0])).toBe(true); words.forEach((w) => expect(d.has(w)).toBe(true)); words.map((w) => w.toLowerCase()).forEach((w) => expect(d.has(w)).toBe(true)); diff --git a/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.ts b/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.ts index 4f2ca589d80..982e01e4fbc 100644 --- a/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.ts +++ b/packages/cspell-lib/src/SpellingDictionary/createSpellingDictionary.ts @@ -3,17 +3,33 @@ import { parseDictionaryLines, buildTrieFast } from 'cspell-trie-lib'; import { SpellingDictionaryFromTrie } from './SpellingDictionaryFromTrie'; import { SpellingDictionary, SpellingDictionaryOptions } from './SpellingDictionary'; import { SpellingDictionaryLoadError } from './SpellingDictionaryError'; +import { operators } from 'gensequence'; export function createSpellingDictionary( wordList: string[] | IterableLike, name: string, source: string, - options?: SpellingDictionaryOptions + options: SpellingDictionaryOptions | undefined ): SpellingDictionary { // console.log(`createSpellingDictionary ${name} ${source}`); const words = parseDictionaryLines(wordList); const trie = buildTrieFast(words); - return new SpellingDictionaryFromTrie(trie, name, options, source); + return new SpellingDictionaryFromTrie(trie, name, options || {}, source); +} + +export function createForbiddenWordsDictionary( + wordList: string[], + name: string, + source: string, + options: SpellingDictionaryOptions | undefined +): SpellingDictionary { + // console.log(`createSpellingDictionary ${name} ${source}`); + const words = parseDictionaryLines(wordList.concat(wordList.map((a) => a.toLowerCase())), { + stripCaseAndAccents: !options?.noSuggest, + }); + const forbidWords = operators.map((w: string) => '!' + w)(words); + const trie = buildTrieFast(forbidWords); + return new SpellingDictionaryFromTrie(trie, name, options || {}, source); } export function createFailedToLoadDictionary(error: SpellingDictionaryLoadError): SpellingDictionary { @@ -23,7 +39,9 @@ export function createFailedToLoadDictionary(error: SpellingDictionaryLoadError) name: options.name, source, type: 'error', + containsNoSuggestWords: false, has: () => false, + isNoSuggestWord: () => false, isForbidden: () => false, suggest: () => [], mapWord: (a) => a, diff --git a/packages/cspell-lib/src/textValidator.test.ts b/packages/cspell-lib/src/textValidator.test.ts index 612e19b3a8f..59f8cd48717 100644 --- a/packages/cspell-lib/src/textValidator.test.ts +++ b/packages/cspell-lib/src/textValidator.test.ts @@ -6,7 +6,7 @@ import { HasWordOptions, ValidationOptions, } from './textValidator'; -import { createCollection } from './SpellingDictionary'; +import { createCollection, getDictionary } from './SpellingDictionary'; import { createSpellingDictionary } from './SpellingDictionary/createSpellingDictionary'; import { FreqCounter } from './util/FreqCounter'; import * as Text from './util/text'; @@ -64,7 +64,7 @@ describe('Validate textValidator functions', () => { }); test('tests trailing s, ed, ing, etc. are attached to the words', async () => { - const dictEmpty = await createSpellingDictionary([], 'empty', 'test'); + const dictEmpty = await createSpellingDictionary([], 'empty', 'test', {}); const text = 'We have PUBLISHed multiple FIXesToThePROBLEMs'; const result = validateText(text, dictEmpty, sToV({})).toArray(); const errors = result.map((wo) => wo.text); @@ -72,14 +72,31 @@ describe('Validate textValidator functions', () => { }); test('tests case in ignore words', async () => { - const dictEmpty = await createSpellingDictionary([], 'empty', 'test'); - const text = 'We have PUBLISHed published multiple FIXesToThePROBLEMs'; + const dict = await getDictionary({ + words: ['=Sample', 'with', 'Issues'], + ignoreWords: ['PUBLISHed', 'FIXesToThePROBLEMs'], // cspell:ignore fixestotheproblems + }); + const text = + 'We have PUBLISHed published multiple FIXesToThePROBLEMs with Sample fixestotheproblems and issues.'; const options: ValidationOptions = { - ignoreWordsAreCaseSensitive: true, - ignoreWords: ['PUBLISHed', 'FIXesToThePROBLEMs'], ignoreCase: false, }; - const result = validateText(text, dictEmpty, options).toArray(); + const result = validateText(text, dict, options).toArray(); + const errors = result.map((wo) => wo.text); + expect(errors).toEqual(['have', 'published', 'multiple', 'fixestotheproblems', 'issues']); + }); + + test('tests case in ignore words ignore case', async () => { + const dict = await getDictionary({ + words: ['=Sample', 'with', 'Issues'], + ignoreWords: ['"PUBLISHed"', 'FIXesToThePROBLEMs'], // cspell:ignore fixestotheproblems + }); + const text = + 'We have PUBLISHed published multiple FIXesToThePROBLEMs with Sample fixestotheproblems and issues.'; + const options: ValidationOptions = { + ignoreCase: true, + }; + const result = validateText(text, dict, options).toArray(); const errors = result.map((wo) => wo.text); expect(errors).toEqual(['have', 'published', 'multiple']); }); @@ -102,7 +119,6 @@ describe('Validate textValidator functions', () => { allowCompoundWords: false, ignoreCase: false, flagWords, - ignoreWordsAreCaseSensitive: true, }; const result = validateText(text, dict, options).toArray(); const errors = result.map((wo) => wo.text); @@ -134,7 +150,7 @@ describe('Validate textValidator functions', () => { }); test('tests maxDuplicateProblems', async () => { - const dict = await createSpellingDictionary([], 'empty', 'test'); + const dict = await createSpellingDictionary([], 'empty', 'test', {}); const text = sampleText; const result = validateText( text, @@ -147,7 +163,7 @@ describe('Validate textValidator functions', () => { const freq = FreqCounter.create(result.map((t) => t.text)); expect(freq.total).toBe(freq.counters.size); const words = freq.counters.keys(); - const dict2 = await createSpellingDictionary(words, 'test', 'test'); + const dict2 = await createSpellingDictionary(words, 'test', 'test', {}); const result2 = [...validateText(text, dict2, sToV({ maxNumberOfProblems: 1000, maxDuplicateProblems: 1 }))]; expect(result2.length).toBe(0); }); @@ -199,16 +215,18 @@ describe('Validate textValidator functions', () => { ${'colour'} | ${[]} | ${[ov({ text: 'colour', isFlagged: true })]} ${'colour'} | ${['colour']} | ${[]} `('Validate forbidden words', ({ text, ignoreWords, expected }) => { - const dict = getSpellingDictionaryCollectionSync(); - const result = [ - ...validateText(text, dict, { ignoreWords, ignoreCase: false, ignoreWordsAreCaseSensitive: false }), - ]; + const dict = getSpellingDictionaryCollectionSync({ ignoreWords }); + const result = [...validateText(text, dict, { ignoreCase: false })]; expect(result).toEqual(expected); }); }); -async function getSpellingDictionaryCollection() { - return getSpellingDictionaryCollectionSync(); +interface WithIgnoreWords { + ignoreWords?: string[]; +} + +async function getSpellingDictionaryCollection(options?: WithIgnoreWords) { + return getSpellingDictionaryCollectionSync(options); } const colors = [ @@ -279,14 +297,18 @@ const sampleText = ` The orange tiger ate the whiteberry and the redberry. `; -function getSpellingDictionaryCollectionSync() { +function getSpellingDictionaryCollectionSync(options?: WithIgnoreWords) { const dicts = [ - createSpellingDictionary(colors, 'colors', 'test'), - createSpellingDictionary(fruit, 'fruit', 'test'), - createSpellingDictionary(animals, 'animals', 'test'), - createSpellingDictionary(insects, 'insects', 'test'), + createSpellingDictionary(colors, 'colors', 'test', {}), + createSpellingDictionary(fruit, 'fruit', 'test', {}), + createSpellingDictionary(animals, 'animals', 'test', {}), + createSpellingDictionary(insects, 'insects', 'test', {}), createSpellingDictionary(words, 'words', 'test', { repMap: [['’', "'"]] }), - createSpellingDictionary(forbiddenWords, 'forbidden-words', 'test'), + createSpellingDictionary(forbiddenWords, 'forbidden-words', 'test', {}), + createSpellingDictionary(options?.ignoreWords || [], 'ignore-words', 'test', { + caseSensitive: true, + noSuggest: true, + }), ]; return createCollection(dicts, 'collection'); diff --git a/packages/cspell-lib/src/textValidator.ts b/packages/cspell-lib/src/textValidator.ts index 930cb561ab9..fbcf9f013bc 100644 --- a/packages/cspell-lib/src/textValidator.ts +++ b/packages/cspell-lib/src/textValidator.ts @@ -5,7 +5,6 @@ import { SpellingDictionary, HasOptions } from './SpellingDictionary/SpellingDic import { Sequence } from 'gensequence'; import * as RxPat from './Settings/RegExpPatterns'; import { split } from './util/wordSplitter'; -import { createSpellingDictionary, SpellingDictionaryCollection } from './SpellingDictionary'; export interface ValidationOptions extends IncludeExcludeOptions { maxNumberOfProblems?: number; @@ -13,17 +12,13 @@ export interface ValidationOptions extends IncludeExcludeOptions { minWordLength?: number; // words to always flag as an error flagWords?: string[]; - ignoreWords?: string[]; allowCompoundWords?: boolean; - /** ignore words are considered case sensitive */ - ignoreWordsAreCaseSensitive: boolean; /** ignore case when checking words against dictionary or ignore words list */ ignoreCase: boolean; } export interface CheckOptions extends ValidationOptions { allowCompoundWords: boolean; - ignoreWordsAreCaseSensitive: boolean; ignoreCase: boolean; } @@ -95,25 +90,15 @@ function lineValidator(dict: SpellingDictionary, options: ValidationOptions): Li const { minWordLength = defaultMinWordLength, flagWords = [], - ignoreWords = [], allowCompoundWords = false, ignoreCase = true, - ignoreWordsAreCaseSensitive: caseSensitive = true, } = options; const hasWordOptions: HasWordOptions = { ignoreCase, useCompounds: allowCompoundWords || undefined, // let the dictionaries decide on useCompounds if allow is false }; - const ignoreDict = createSpellingDictionary(ignoreWords, '__ignore words__', 'ignore words', { - caseSensitive, - }); - - const dictCol = new SpellingDictionaryCollection([dict, ignoreDict], dict.name, []); - - function isWordIgnored(word: string) { - return ignoreDict.has(word, { ignoreCase }); - } + const dictCol = dict; const setOfFlagWords = new Set(flagWords); const setOfKnownSuccessfulWords = new Set(); @@ -135,6 +120,10 @@ function lineValidator(dict: SpellingDictionary, options: ValidationOptions): Li return setOfFlagWords.has(text) || setOfFlagWords.has(text.toLowerCase()) || dictCol.isForbidden(text); } + function isWordIgnored(word: string): boolean { + return dict.isNoSuggestWord(word, options); + } + function checkFlagWords(word: ValidationResult): ValidationResult { const isIgnored = isWordIgnored(word.text); const isFlagged = !isIgnored && testForFlaggedWord(word); diff --git a/packages/cspell-lib/src/validator.ts b/packages/cspell-lib/src/validator.ts index d111c071ac5..28c39f5249a 100644 --- a/packages/cspell-lib/src/validator.ts +++ b/packages/cspell-lib/src/validator.ts @@ -41,7 +41,6 @@ export async function validateText( export function settingsToValidateOptions(settings: CSpellUserSettings): TV.ValidationOptions { const opt: TV.ValidationOptions = { ...settings, - ignoreWordsAreCaseSensitive: settings.caseSensitive ?? true, ignoreCase: !(settings.caseSensitive ?? false), }; return opt; diff --git a/packages/cspell-trie-lib/src/lib/SimpleDictionaryParser.test.ts b/packages/cspell-trie-lib/src/lib/SimpleDictionaryParser.test.ts index 0d4662901ac..31c65a289c2 100644 --- a/packages/cspell-trie-lib/src/lib/SimpleDictionaryParser.test.ts +++ b/packages/cspell-trie-lib/src/lib/SimpleDictionaryParser.test.ts @@ -45,6 +45,36 @@ describe('Validate SimpleDictionaryParser', () => { ]); }); + test('Auto generate cases', () => { + const words = ['!forbid', '*End', '+Middle+', 'Begin*', 'Café']; + const expected = [ + '!forbid', + '+End', + '+Middle+', + 'Begin', + 'Begin+', + 'Café', + 'End', + '~+end', + '~+middle+', + '~begin', + '~begin+', + '~cafe', + '~café', + '~end', + ]; + const trie = parseDictionary(words.join('\n')); + const result = [...trie.words()]; + expect(result).toEqual(expected); + }); + + test('preserve cases', () => { + const words = ['!forbid', '+End', '+Middle+', 'Begin', 'Begin+', 'Café', 'End']; + const trie = parseDictionary(words.join('\n'), { stripCaseAndAccents: false }); + const result = [...trie.words()]; + expect(result).toEqual(words); + }); + function toL(a: string): string { return a.toLowerCase(); } @@ -87,15 +117,36 @@ describe('Validate SimpleDictionaryParser', () => { } test.each` - lines | expected - ${s('word')} | ${s('word')} - ${s('two-word')} | ${s('two-word')} - ${s('Word')} | ${s('Word|~word')} - ${s('*error*')} | ${s('error|error+|+error|+error+')} + lines | expected + ${s('word')} | ${s('word')} + ${s('two-word')} | ${s('two-word')} + ${s('Geschäft')} | ${s('Geschäft|~geschäft|~geschaft')} + ${s('=Geschäft')} | ${s('Geschäft')} + ${s('"Geschäft"')} | ${s('Geschäft')} + ${s('="Geschäft"')} | ${s('Geschäft')} + ${s('Word')} | ${s('Word|~word')} + ${s('*error*')} | ${s('error|error+|+error|+error+')} `('parseDictionaryLines simple $lines', ({ lines, expected }) => { const r = [...parseDictionaryLines(lines)]; expect(r).toEqual(expected); }); + + // cspell:ignore érror + test.each` + lines | expected + ${s('word')} | ${s('word')} + ${s('two-word')} | ${s('two-word')} + ${s('Geschäft')} | ${s('Geschäft')} + ${s('=Geschäft')} | ${s('Geschäft')} + ${s('"Geschäft"')} | ${s('Geschäft')} + ${s('="Geschäft"')} | ${s('Geschäft')} + ${s('Word')} | ${s('Word')} + ${s('*error*')} | ${s('error|error+|+error|+error+')} + ${s('*érror*')} | ${s('érror|érror+|+érror|+érror+')} + `('parseDictionaryLines simple no strip $lines', ({ lines, expected }) => { + const r = [...parseDictionaryLines(lines, { stripCaseAndAccents: false })]; + expect(r).toEqual(expected); + }); }); function dictionary() { diff --git a/packages/cspell-trie-lib/src/lib/SimpleDictionaryParser.ts b/packages/cspell-trie-lib/src/lib/SimpleDictionaryParser.ts index dce09f72d5e..d9c070a7322 100644 --- a/packages/cspell-trie-lib/src/lib/SimpleDictionaryParser.ts +++ b/packages/cspell-trie-lib/src/lib/SimpleDictionaryParser.ts @@ -1,6 +1,13 @@ import { operators } from 'gensequence'; import { normalizeWord, normalizeWordForCaseInsensitive } from './util'; -import { COMPOUND_FIX, OPTIONAL_COMPOUND_FIX, FORBID_PREFIX, CASE_INSENSITIVE_PREFIX, LINE_COMMENT } from './constants'; +import { + COMPOUND_FIX, + OPTIONAL_COMPOUND_FIX, + FORBID_PREFIX, + CASE_INSENSITIVE_PREFIX, + LINE_COMMENT, + IDENTITY_PREFIX, +} from './constants'; import { Trie } from './trie'; import { buildTrieFast } from './TrieBuilder'; @@ -9,7 +16,21 @@ export interface ParseDictionaryOptions { optionalCompoundCharacter: string; forbiddenPrefix: string; caseInsensitivePrefix: string; + /** + * Start of a single-line comment. + */ commentCharacter: string; + + /** + * if word starts with prefix, do not strip case or accents. + * Prefix is not stored. + */ + keepExactPrefix: string; + + /** + * Tell the parser to NOT automatically strip case and accents. + */ + stripCaseAndAccents: boolean; } const _defaultOptions: ParseDictionaryOptions = { @@ -18,6 +39,8 @@ const _defaultOptions: ParseDictionaryOptions = { compoundCharacter: COMPOUND_FIX, forbiddenPrefix: FORBID_PREFIX, caseInsensitivePrefix: CASE_INSENSITIVE_PREFIX, + keepExactPrefix: IDENTITY_PREFIX, + stripCaseAndAccents: true, }; export const defaultParseDictionaryOptions: ParseDictionaryOptions = Object.freeze(_defaultOptions); @@ -26,20 +49,23 @@ export const defaultParseDictionaryOptions: ParseDictionaryOptions = Object.free * Normalizes a dictionary words based upon prefix / suffixes. * Case insensitive versions are also generated. * @param lines - one word per line - * @param options - defines prefixes used when parsing lines. + * @param _options - defines prefixes used when parsing lines. * @returns words that have been normalized. */ export function parseDictionaryLines( lines: Iterable, - options: ParseDictionaryOptions = _defaultOptions + options?: Partial ): Iterable { + const _options = mergeOptions(_defaultOptions, options); const { commentCharacter, optionalCompoundCharacter: optionalCompound, compoundCharacter: compound, caseInsensitivePrefix: ignoreCase, forbiddenPrefix: forbidden, - } = options; + keepExactPrefix: keepCase, + stripCaseAndAccents, + } = _options; const regexComment = new RegExp(escapeRegEx(commentCharacter) + '.*', 'g'); @@ -80,19 +106,28 @@ export function parseDictionaryLines( } } - const doNotNormalizePrefix = new Set([forbidden, ignoreCase]); + const doNotNormalizePrefix = new Set([forbidden, ignoreCase, keepCase, '"']); function removeDoublePrefix(w: string): string { return w.startsWith(ignoreCase + ignoreCase) ? w.slice(1) : w; } + function stripKeepCasePrefixAndQuotes(word: string): string { + word = word.replace(/"(.*)"/, '$1'); + return word[0] === keepCase ? word.slice(1) : word; + } + + function _normalize(word: string): string { + return normalizeWord(stripKeepCasePrefixAndQuotes(word)); + } + function* mapNormalize(word: string) { - word = normalizeWord(word); + const nWord = _normalize(word); const forms = new Set(); - forms.add(word); - if (!doNotNormalizePrefix.has(word[0])) { - for (const n of normalizeWordForCaseInsensitive(word)) { - if (n !== word) forms.add(ignoreCase + n); + forms.add(nWord); + if (stripCaseAndAccents && !doNotNormalizePrefix.has(word[0])) { + for (const n of normalizeWordForCaseInsensitive(nWord)) { + if (n !== nWord) forms.add(ignoreCase + n); } } yield* forms; @@ -112,22 +147,32 @@ export function parseDictionaryLines( return processLines(lines); } -export function parseLinesToDictionary( - lines: Iterable, - options: ParseDictionaryOptions = _defaultOptions -): Trie { - const dictLines = parseDictionaryLines(lines, options); +export function parseLinesToDictionary(lines: Iterable, options?: Partial): Trie { + const _options = mergeOptions(_defaultOptions, options); + const dictLines = parseDictionaryLines(lines, _options); return buildTrieFast([...new Set(dictLines)].sort(), { - compoundCharacter: options.compoundCharacter, - forbiddenWordPrefix: options.forbiddenPrefix, - stripCaseAndAccentsPrefix: options.caseInsensitivePrefix, + compoundCharacter: _options.compoundCharacter, + forbiddenWordPrefix: _options.forbiddenPrefix, + stripCaseAndAccentsPrefix: _options.caseInsensitivePrefix, }); } -export function parseDictionary(text: string, options?: ParseDictionaryOptions): Trie { +export function parseDictionary(text: string, options?: Partial): Trie { return parseLinesToDictionary(text.split('\n'), options); } function escapeRegEx(s: string) { return s.replace(/[|\\{}()[\]^$+*?.]/g, '\\$&').replace(/-/g, '\\x2d'); } + +function mergeOptions( + base: ParseDictionaryOptions, + ...partials: (Partial | undefined)[] +): ParseDictionaryOptions { + const opt: ParseDictionaryOptions = { ...base }; + for (const p of partials) { + if (!p) continue; + Object.assign(opt, p); + } + return opt; +} diff --git a/packages/cspell-trie-lib/src/lib/constants.ts b/packages/cspell-trie-lib/src/lib/constants.ts index 77103242ded..0dac868ff71 100644 --- a/packages/cspell-trie-lib/src/lib/constants.ts +++ b/packages/cspell-trie-lib/src/lib/constants.ts @@ -5,6 +5,7 @@ export const OPTIONAL_COMPOUND_FIX = '*'; export const CASE_INSENSITIVE_PREFIX = '~'; export const FORBID_PREFIX = '!'; export const LINE_COMMENT = '#'; +export const IDENTITY_PREFIX = '='; export const defaultTrieOptions: TrieOptions = Object.freeze({ compoundCharacter: COMPOUND_FIX, diff --git a/packages/cspell-types/cspell.schema.json b/packages/cspell-types/cspell.schema.json index a4d551b7b71..a64cdd37c5f 100644 --- a/packages/cspell-types/cspell.schema.json +++ b/packages/cspell-types/cspell.schema.json @@ -45,7 +45,11 @@ }, "name": { "$ref": "#/definitions/DictionaryId", - "description": "The reference name of the dictionary, used with program language settings" + "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`" + }, + "noSuggest": { + "description": "Indicate that suggestions should not come from this dictionary. Words in this dictionary are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in this dictionary, it will be removed from the set of possible suggestions.", + "type": "boolean" }, "repMap": { "$ref": "#/definitions/ReplaceMap", @@ -76,7 +80,11 @@ }, "name": { "$ref": "#/definitions/DictionaryId", - "description": "The reference name of the dictionary, used with program language settings" + "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`" + }, + "noSuggest": { + "description": "Indicate that suggestions should not come from this dictionary. Words in this dictionary are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in this dictionary, it will be removed from the set of possible suggestions.", + "type": "boolean" }, "path": { "$ref": "#/definitions/CustomDictionaryPath", @@ -121,7 +129,11 @@ }, "name": { "$ref": "#/definitions/DictionaryId", - "description": "The reference name of the dictionary, used with program language settings" + "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`" + }, + "noSuggest": { + "description": "Indicate that suggestions should not come from this dictionary. Words in this dictionary are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in this dictionary, it will be removed from the set of possible suggestions.", + "type": "boolean" }, "path": { "$ref": "#/definitions/DictionaryPath", @@ -143,7 +155,13 @@ "type": "object" }, "DictionaryId": { - "description": "This matches the name in a dictionary definition", + "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`", + "pattern": "^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$", + "type": "string" + }, + "DictionaryNegRef": { + "description": "This a negative reference to a named dictionary.\n\nIt is used to exclude or include a dictionary by name.\n\nThe reference starts with 1 or more `!`.\n- `!` - Used to exclude the dictionary matching ``\n- `!!` - Used to re-include a dictionary matching `` Overrides `!`.\n- `!!!` - Used to exclude a dictionary matching `` Overrides `!!`.", + "pattern": "^(?=!+[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$", "type": "string" }, "DictionaryPath": { @@ -151,6 +169,22 @@ "pattern": "^.*\\.(?:txt|trie)(?:\\.gz)?$", "type": "string" }, + "DictionaryRef": { + "$ref": "#/definitions/DictionaryId", + "description": "This a reference to a named dictionary. It is expected to match the name of a dictionary.", + "pattern": "^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$" + }, + "DictionaryReference": { + "anyOf": [ + { + "$ref": "#/definitions/DictionaryRef" + }, + { + "$ref": "#/definitions/DictionaryNegRef" + } + ], + "description": "Reference to a dictionary by name. One of:\n- {@link DictionaryRef } \n- {@link DictionaryNegRef }" + }, "FsPath": { "description": "A File System Path", "type": "string" @@ -205,9 +239,9 @@ "type": "string" }, "dictionaries": { - "description": "Optional list of dictionaries to use.", + "description": "Optional list of dictionaries to use. Each entry should match the name of the dictionary. To remove a dictionary from the list add `!` before the name. i.e. `!typescript` will turn of the dictionary with the name `typescript`.", "items": { - "$ref": "#/definitions/DictionaryId" + "$ref": "#/definitions/DictionaryReference" }, "type": "array" }, @@ -289,6 +323,13 @@ "description": "Optional name of configuration", "type": "string" }, + "noSuggestDictionaries": { + "description": "Optional list of dictionaries that will not be used for suggestions. Words in these dictionaries are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in one of these dictionaries, it will be removed from the set of possible suggestions.", + "items": { + "$ref": "#/definitions/DictionaryReference" + }, + "type": "array" + }, "patterns": { "description": "Defines a list of patterns that can be used in ignoreRegExpList and includeRegExpList", "items": { @@ -331,9 +372,9 @@ "type": "string" }, "dictionaries": { - "description": "Optional list of dictionaries to use.", + "description": "Optional list of dictionaries to use. Each entry should match the name of the dictionary. To remove a dictionary from the list add `!` before the name. i.e. `!typescript` will turn of the dictionary with the name `typescript`.", "items": { - "$ref": "#/definitions/DictionaryId" + "$ref": "#/definitions/DictionaryReference" }, "type": "array" }, @@ -440,6 +481,13 @@ "description": "Optional name of configuration", "type": "string" }, + "noSuggestDictionaries": { + "description": "Optional list of dictionaries that will not be used for suggestions. Words in these dictionaries are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in one of these dictionaries, it will be removed from the set of possible suggestions.", + "items": { + "$ref": "#/definitions/DictionaryReference" + }, + "type": "array" + }, "numSuggestions": { "default": 10, "description": "Number of suggestions to make", @@ -629,9 +677,9 @@ "type": "string" }, "dictionaries": { - "description": "Optional list of dictionaries to use.", + "description": "Optional list of dictionaries to use. Each entry should match the name of the dictionary. To remove a dictionary from the list add `!` before the name. i.e. `!typescript` will turn of the dictionary with the name `typescript`.", "items": { - "$ref": "#/definitions/DictionaryId" + "$ref": "#/definitions/DictionaryReference" }, "type": "array" }, @@ -762,6 +810,13 @@ "description": "Prevents searching for local configuration when checking individual documents.", "type": "boolean" }, + "noSuggestDictionaries": { + "description": "Optional list of dictionaries that will not be used for suggestions. Words in these dictionaries are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in one of these dictionaries, it will be removed from the set of possible suggestions.", + "items": { + "$ref": "#/definitions/DictionaryReference" + }, + "type": "array" + }, "numSuggestions": { "default": 10, "description": "Number of suggestions to make", diff --git a/packages/cspell-types/src/settings/CSpellSettingsDef.ts b/packages/cspell-types/src/settings/CSpellSettingsDef.ts index 310d022eae6..8c2d6845e5b 100644 --- a/packages/cspell-types/src/settings/CSpellSettingsDef.ts +++ b/packages/cspell-types/src/settings/CSpellSettingsDef.ts @@ -272,8 +272,24 @@ export interface BaseSetting { /** Define additional available dictionaries */ dictionaryDefinitions?: DictionaryDefinition[]; - /** Optional list of dictionaries to use. */ - dictionaries?: DictionaryId[]; + /** + * Optional list of dictionaries to use. + * Each entry should match the name of the dictionary. + * To remove a dictionary from the list add `!` before the name. + * i.e. `!typescript` will turn of the dictionary with the name `typescript`. + */ + dictionaries?: DictionaryReference[]; + + /** + * Optional list of dictionaries that will not be used for suggestions. + * Words in these dictionaries are considered correct, but will not be + * used when making spell correction suggestions. + * + * Note: if a word is suggested by another dictionary, but found in + * one of these dictionaries, it will be removed from the set of + * possible suggestions. + */ + noSuggestDictionaries?: DictionaryReference[]; /** * List of RegExp patterns or Pattern names to exclude from spell checking. @@ -301,7 +317,16 @@ export type DictionaryDefinition = | DictionaryDefinitionLegacy; export interface DictionaryDefinitionBase { - /** The reference name of the dictionary, used with program language settings */ + /** + * This is the name of a dictionary. + * + * Name Format: + * - Must contain at least 1 number or letter. + * - spaces are allowed. + * - Leading and trailing space will be removed. + * - Names ARE case-sensitive + * - Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~` + */ name: DictionaryId; /** Optional description */ description?: string; @@ -309,6 +334,16 @@ export interface DictionaryDefinitionBase { repMap?: ReplaceMap; /** Use Compounds */ useCompounds?: boolean; + /** + * Indicate that suggestions should not come from this dictionary. + * Words in this dictionary are considered correct, but will not be + * used when making spell correction suggestions. + * + * Note: if a word is suggested by another dictionary, but found in + * this dictionary, it will be removed from the set of + * possible suggestions. + */ + noSuggest?: boolean; } export interface DictionaryDefinitionPreferred extends DictionaryDefinitionBase { @@ -458,9 +493,52 @@ export type PatternRef = Pattern | PatternId | PredefinedPatterns; /** A list of pattern names or regular expressions */ export type RegExpPatternList = PatternRef[]; -/** This matches the name in a dictionary definition */ +/** + * This is the name of a dictionary. + * + * Name Format: + * - Must contain at least 1 number or letter. + * - spaces are allowed. + * - Leading and trailing space will be removed. + * - Names ARE case-sensitive + * - Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~` + * + * @pattern ^(?=[^!*,;{}[\]~\n]+$)(?=(.*\w)).+$ + */ export type DictionaryId = string; +/** + * This a reference to a named dictionary. + * It is expected to match the name of a dictionary. + * + * @pattern ^(?=[^!*,;{}[\]~\n]+$)(?=(.*\w)).+$ + */ +export type DictionaryRef = DictionaryId; + +/** + * This a negative reference to a named dictionary. + * + * It is used to exclude or include a dictionary by name. + * + * The reference starts with 1 or more `!`. + * - `!` - Used to exclude the dictionary matching `` + * - `!!` - Used to re-include a dictionary matching `` + * Overrides `!`. + * - `!!!` - Used to exclude a dictionary matching `` + * Overrides `!!`. + * + * @pattern ^(?=!+[^!*,;{}[\]~\n]+$)(?=(.*\w)).+$ + */ +export type DictionaryNegRef = string; + +/** + * Reference to a dictionary by name. + * One of: + * - {@link DictionaryRef} + * - {@link DictionaryNegRef} + */ +export type DictionaryReference = DictionaryRef | DictionaryNegRef; + /** This is a written language locale like: 'en', 'en-GB', 'fr', 'es', 'de', etc. */ export type LocaleId = string;