feat: Add support for noSuggest dictionaries. (#1554)

* dev: add `noSuggestDictionaries` and `noSuggest` dictionary options.
* dev: add some samples
* test: Code coverage for Cache
* dev: [trie-lib] add option to control adding normalized versions of words.
* dev: correctly merge `noSuggestDictionaries`
* dev: Integrate `isNoSuggestWord` and `isForbidden` into `SpellingDictionary`
* dev: `@pattern` is not allowed on a non-string property
* fix: Make sure the case setting is passed through to no suggest tests.

.prettierignore

@@ -1,7 +1,8 @@
 .vscode/
 [sS]amples/
 [tT]emp/
-**/dist/
+**/dist/**
+**/node_modules/**
 **/package-lock.json
 CHANGELOG.md
 coverage

cspell.schema.json

@@ -45,7 +45,11 @@
         },
         "name": {
           "$ref": "#/definitions/DictionaryId",
-          "description": "The reference name of the dictionary, used with program language settings"
+          "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`"
+        },
+        "noSuggest": {
+          "description": "Indicate that suggestions should not come from this dictionary. Words in this dictionary are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in this dictionary, it will be removed from the set of possible suggestions.",
+          "type": "boolean"
         },
         "repMap": {
           "$ref": "#/definitions/ReplaceMap",
@@ -76,7 +80,11 @@
         },
         "name": {
           "$ref": "#/definitions/DictionaryId",
-          "description": "The reference name of the dictionary, used with program language settings"
+          "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`"
+        },
+        "noSuggest": {
+          "description": "Indicate that suggestions should not come from this dictionary. Words in this dictionary are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in this dictionary, it will be removed from the set of possible suggestions.",
+          "type": "boolean"
         },
         "path": {
           "$ref": "#/definitions/CustomDictionaryPath",
@@ -121,7 +129,11 @@
         },
         "name": {
           "$ref": "#/definitions/DictionaryId",
-          "description": "The reference name of the dictionary, used with program language settings"
+          "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`"
+        },
+        "noSuggest": {
+          "description": "Indicate that suggestions should not come from this dictionary. Words in this dictionary are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in this dictionary, it will be removed from the set of possible suggestions.",
+          "type": "boolean"
         },
         "path": {
           "$ref": "#/definitions/DictionaryPath",
@@ -143,14 +155,36 @@
       "type": "object"
     },
     "DictionaryId": {
-      "description": "This matches the name in a dictionary definition",
+      "description": "This is the name of a dictionary.\n\nName Format:\n- Must contain at least 1 number or letter.\n- spaces are allowed.\n- Leading and trailing space will be removed.\n- Names ARE case-sensitive\n- Must not contain `*`, `!`, `;`, `,`, `{`, `}`, `[`, `]`, `~`",
+      "pattern": "^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$",
+      "type": "string"
+    },
+    "DictionaryNegRef": {
+      "description": "This a negative reference to a named dictionary.\n\nIt is used to exclude or include a dictionary by name.\n\nThe reference starts with 1 or more `!`.\n- `!<dictionary_name>` - Used to exclude the dictionary matching `<dictionary_name>`\n- `!!<dictionary_name>` - Used to re-include a dictionary matching `<dictionary_name>`    Overrides `!<dictionary_name>`.\n- `!!!<dictionary_name>` - Used to exclude a dictionary matching `<dictionary_name>`    Overrides `!!<dictionary_name>`.",
+      "pattern": "^(?=!+[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$",
       "type": "string"
     },
     "DictionaryPath": {
       "description": "A File System Path to a dictionary file.",
       "pattern": "^.*\\.(?:txt|trie)(?:\\.gz)?$",
       "type": "string"
     },
+    "DictionaryRef": {
+      "$ref": "#/definitions/DictionaryId",
+      "description": "This a reference to a named dictionary. It is expected to match the name of a dictionary.",
+      "pattern": "^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$"
+    },
+    "DictionaryReference": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/DictionaryRef"
+        },
+        {
+          "$ref": "#/definitions/DictionaryNegRef"
+        }
+      ],
+      "description": "Reference to a dictionary by name. One of:\n-  {@link  DictionaryRef } \n-  {@link  DictionaryNegRef }"
+    },
     "FsPath": {
       "description": "A File System Path",
       "type": "string"
@@ -205,9 +239,9 @@
           "type": "string"
         },
         "dictionaries": {
-          "description": "Optional list of dictionaries to use.",
+          "description": "Optional list of dictionaries to use. Each entry should match the name of the dictionary. To remove a dictionary from the list add `!` before the name. i.e. `!typescript` will turn of the dictionary with the name `typescript`.",
           "items": {
-            "$ref": "#/definitions/DictionaryId"
+            "$ref": "#/definitions/DictionaryReference"
           },
           "type": "array"
         },
@@ -289,6 +323,13 @@
           "description": "Optional name of configuration",
           "type": "string"
         },
+        "noSuggestDictionaries": {
+          "description": "Optional list of dictionaries that will not be used for suggestions. Words in these dictionaries are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in one of these dictionaries, it will be removed from the set of possible suggestions.",
+          "items": {
+            "$ref": "#/definitions/DictionaryReference"
+          },
+          "type": "array"
+        },
         "patterns": {
           "description": "Defines a list of patterns that can be used in ignoreRegExpList and includeRegExpList",
           "items": {
@@ -331,9 +372,9 @@
           "type": "string"
         },
         "dictionaries": {
-          "description": "Optional list of dictionaries to use.",
+          "description": "Optional list of dictionaries to use. Each entry should match the name of the dictionary. To remove a dictionary from the list add `!` before the name. i.e. `!typescript` will turn of the dictionary with the name `typescript`.",
           "items": {
-            "$ref": "#/definitions/DictionaryId"
+            "$ref": "#/definitions/DictionaryReference"
           },
           "type": "array"
         },
@@ -440,6 +481,13 @@
           "description": "Optional name of configuration",
           "type": "string"
         },
+        "noSuggestDictionaries": {
+          "description": "Optional list of dictionaries that will not be used for suggestions. Words in these dictionaries are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in one of these dictionaries, it will be removed from the set of possible suggestions.",
+          "items": {
+            "$ref": "#/definitions/DictionaryReference"
+          },
+          "type": "array"
+        },
         "numSuggestions": {
           "default": 10,
           "description": "Number of suggestions to make",
@@ -629,9 +677,9 @@
       "type": "string"
     },
     "dictionaries": {
-      "description": "Optional list of dictionaries to use.",
+      "description": "Optional list of dictionaries to use. Each entry should match the name of the dictionary. To remove a dictionary from the list add `!` before the name. i.e. `!typescript` will turn of the dictionary with the name `typescript`.",
       "items": {
-        "$ref": "#/definitions/DictionaryId"
+        "$ref": "#/definitions/DictionaryReference"
       },
       "type": "array"
     },
@@ -762,6 +810,13 @@
       "description": "Prevents searching for local configuration when checking individual documents.",
       "type": "boolean"
     },
+    "noSuggestDictionaries": {
+      "description": "Optional list of dictionaries that will not be used for suggestions. Words in these dictionaries are considered correct, but will not be used when making spell correction suggestions.\n\nNote: if a word is suggested by another dictionary, but found in one of these dictionaries, it will be removed from the set of possible suggestions.",
+      "items": {
+        "$ref": "#/definitions/DictionaryReference"
+      },
+      "type": "array"
+    },
     "numSuggestions": {
       "default": 10,
       "description": "Number of suggestions to make",

packages/cspell-lib/samples/.cspell.json

@@ -47,7 +47,7 @@
             "languageId": "c,cpp",
             // Language locale. i.e. en-US, de-AT, or ru. * will match all locales.
             // Multiple locales can be specified like: "en, en-US" to match both English and English US.
-            "local": "*",
+            "locale": "*",
             // By default the whole text of a file is included for spell checking
             // Adding patterns to the "includeRegExpList" to only include matching patterns
             "includeRegExpList": [

packages/cspell-lib/samples/configurations/.cspell/custom-dictionary.txt

@@ -0,0 +1 @@
+# Custom Dictionary Words

packages/cspell-lib/samples/configurations/.cspell/ignore-words-2.txt

@@ -0,0 +1,2 @@
+# List of words to be ignored.
+nosugg2

packages/cspell-lib/samples/configurations/.cspell/ignore-words.txt

@@ -0,0 +1,2 @@
+# List of words to be ignored.
+nosugg1

packages/cspell-lib/samples/configurations/README.md

@@ -0,0 +1,7 @@
+# Sample CSpell Configurations
+
+- [cspell-dictionaries](./cspell-dictionaries.json) - example of loading custom dictionaries.
+
+nosugg1
+
+nosugg2

packages/cspell-lib/samples/configurations/cspell-dictionaries.json

@@ -0,0 +1,33 @@
+{
+    "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json",
+    "version": "0.2",
+    "dictionaries": [
+        "typescript",
+        "custom-dictionary",
+        "ignore-words"
+    ],
+    "noSuggestDictionaries": [
+        "ignore-words-2"
+    ],
+    "import": [
+        "@cspell/dict-python/cspell-ext.json"
+    ],
+    "dictionaryDefinitions": [
+        {
+            "name": "custom-dictionary",
+            "path": "./.cspell/custom-dictionary.txt",
+            "addWords": true
+        },
+        {
+            "name": "ignore-words",
+            "path": "./.cspell/ignore-words.txt",
+            "addWords": true,
+            "noSuggest": true
+        },
+        {
+            "name": "ignore-words-2",
+            "path": "./.cspell/ignore-words-2.txt",
+            "addWords": true
+        }
+    ]
+}

packages/cspell-lib/samples/configurations/cspell.json

@@ -0,0 +1,6 @@
+{
+    "flagWords": [],
+    "import": [
+        "./cspell-dictionaries.json"
+    ]
+}

packages/cspell-lib/samples/yaml-config/cspell.yaml

@@ -1,5 +1,5 @@
 id: Yaml Example Config
 import:
-  - ./../cspell-imports.json
+  - ./../cspell-imports.json # Note this one does not exist on purpose.
   - ./../cspell-includes.json
   - ../.cspell.json

packages/cspell-lib/src/Cache/cspell.cache.test.ts

@@ -0,0 +1,9 @@
+import { IssueCode } from './cspell.cache';
+
+describe('Cache', () => {
+    test('IssueCode', () => {
+        const codes = [IssueCode.UnknownWord, IssueCode.ForbiddenWord, IssueCode.KnownIssue];
+        const sum = codes.reduce((a, b) => a + b, 0);
+        expect(sum).toBe(IssueCode.ALL);
+    });
+});

packages/cspell-lib/src/Cache/cspell.cache.ts

@@ -9,38 +9,39 @@ export interface CSpellCache {
     files: CachedFile[];
 }
 
-type Version = '0.1';
+export type Version = '0.1';
 
 /**
  * Hash used. Starts with hash id. i.e. `sha1-` or `sha512-`.
  */
-type Hash = string;
+export type Hash = string;
 
-type UriRelPath = string;
+export type UriRelPath = string;
 
-enum IssueCode {
+export enum IssueCode {
     UnknownWord = 1 << 0,
     ForbiddenWord = 1 << 1,
     KnownIssue = 1 << 2,
+    ALL = IssueCode.UnknownWord | IssueCode.ForbiddenWord | IssueCode.KnownIssue,
 }
 
-interface CachedFile {
+export interface CachedFile {
     hash: Hash;
     path: UriRelPath;
     issues: Issue[];
 }
 
-type Issue = IssueEntry | IssueLine;
+export type Issue = IssueEntry | IssueLine;
 
-interface IssueEntry {
+export interface IssueEntry {
     line: number;
     character: number;
     code: IssueCode;
     text: string;
     len: number;
 }
 
-type IssueLine = [
+export type IssueLine = [
     line: IssueEntry['line'],
     character: IssueEntry['character'],
     code: IssueEntry['code'],

packages/cspell-lib/src/Cache/index.test.ts

@@ -0,0 +1,7 @@
+import {} from './index';
+
+describe('index', () => {
+    test('index', () => {
+        expect(true).toBe(true);
+    });
+});

packages/cspell-lib/src/Cache/index.ts

@@ -0,0 +1 @@
+export type { CSpellCache } from './cspell.cache';

packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts

@@ -20,17 +20,22 @@ import {
 } from './CSpellSettingsServer';
 import { getDefaultSettings, _defaultSettings } from './DefaultSettings';
 import { CSpellSettingsWithSourceTrace, CSpellUserSettings, ImportFileRef } from '@cspell/cspell-types';
+import { logError, logWarning } from '../util/logger';
+import { mocked } from 'ts-jest/utils';
 import * as path from 'path';
 import { URI } from 'vscode-uri';
 
-const { normalizeSettings } = __testing__;
+const { normalizeSettings, validateRawConfigVersion, validateRawConfigExports } = __testing__;
 
 const rootCspellLib = path.resolve(path.join(__dirname, '../..'));
 const samplesDir = path.resolve(rootCspellLib, 'samples');
 const samplesSrc = path.join(samplesDir, 'src');
 
 jest.mock('../util/logger');
 
+const mockedLogError = mocked(logError);
+const mockedLogWarning = mocked(logWarning);
+
 describe('Validate CSpellSettingsServer', () => {
     test('tests mergeSettings with conflicting "name"', () => {
         const left = { name: 'Left' };
@@ -415,6 +420,11 @@ describe('Validate Glob resolution', () => {
 });
 
 describe('Validate search/load config files', () => {
+    beforeEach(() => {
+        mockedLogError.mockClear();
+        mockedLogWarning.mockClear();
+    });
+
     function importRefWithError(filename: string): ImportFileRefWithError {
         return {
             filename,
@@ -537,6 +547,24 @@ describe('Validate search/load config files', () => {
         const result = await loadConfig(uriYarn2TestMedCspell, {});
         expect(result.dictionaries).toEqual(['medical terms']);
     });
+
+    test.each`
+        config                  | mocked              | expected
+        ${{ version: 'hello' }} | ${mockedLogError}   | ${'Unsupported config file version: "hello"\n  File: "filename"'}
+        ${{ version: '0.1' }}   | ${mockedLogWarning} | ${'Legacy config file version found: "0.1", upgrade to "0.2"\n  File: "filename"'}
+        ${{ version: '0.3' }}   | ${mockedLogWarning} | ${'Newer config file version found: "0.3". Supported version is "0.2"\n  File: "filename"'}
+    `('validateRawConfigVersion $config', ({ config, mocked, expected }) => {
+        validateRawConfigVersion(config, { filename: 'filename' });
+        expect(mocked).toHaveBeenCalledWith(expected);
+    });
+
+    test('validateRawConfigExports', () => {
+        const d = { default: {}, name: '' };
+        const c: CSpellUserSettings = d;
+        expect(() => validateRawConfigExports(c, { filename: 'filename' })).toThrowError(
+            'Module `export default` is not supported.\n  Use `module.exports =` instead.\n  File: "filename"'
+        );
+    });
 });
 
 function oc<T>(v: Partial<T>): T {