From c7ffcc786ed360fc1a59f84915ea7d204d51d3a5 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Sat, 11 Sep 2021 17:47:57 +0200 Subject: [PATCH] fix: drop need for iconv-lite and iterable-to-stream (#1677) - iterable-to-stream - was replaced with the now built-in method. - iconv-lite - was never really needed since all files are expected to be in `utf-8`. - Bump the minimum version of not to the original LTS version 12.13.0 - cspell-io - remove unused async line reader and other unused async functions. NOTE: This might be a breaking change for any external party that depends upon cspell-io. --- packages/cspell-bundled-dicts/package.json | 2 +- packages/cspell-dynamic-loader/package.json | 2 +- packages/cspell-glob/package.json | 2 +- packages/cspell-io/package-lock.json | 28 +--- packages/cspell-io/package.json | 7 +- .../cspell-io/src/async/asyncIterable.test.ts | 14 ++ .../cspell-io/src/file/fileReader.test.ts | 79 +---------- packages/cspell-io/src/file/fileReader.ts | 127 +----------------- packages/cspell-io/src/file/fileWriter.ts | 3 +- packages/cspell-io/src/index.test.ts | 7 + packages/cspell-lib/package.json | 2 +- packages/cspell-tools/package.json | 2 +- packages/cspell-trie-lib/package.json | 2 +- packages/cspell-trie/package-lock.json | 5 - packages/cspell-trie/package.json | 5 +- packages/cspell-trie/src/app.ts | 6 +- packages/cspell-trie2-lib/package.json | 2 +- packages/cspell-types/package.json | 2 +- packages/cspell/package.json | 2 +- packages/hunspell-reader/package.json | 2 +- test-packages/test-cspell-io/src/index.ts | 2 +- 21 files changed, 51 insertions(+), 252 deletions(-) create mode 100644 packages/cspell-io/src/async/asyncIterable.test.ts create mode 100644 packages/cspell-io/src/index.test.ts diff --git a/packages/cspell-bundled-dicts/package.json b/packages/cspell-bundled-dicts/package.json index 906b3c487a5..e3322260cbb 100644 --- a/packages/cspell-bundled-dicts/package.json +++ b/packages/cspell-bundled-dicts/package.json @@ -80,7 +80,7 @@ "@cspell/dict-typescript": "^1.0.19" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@cspell/cspell-tools": "^5.9.0", diff --git a/packages/cspell-dynamic-loader/package.json b/packages/cspell-dynamic-loader/package.json index 78c876d607d..f0d13e494d5 100644 --- a/packages/cspell-dynamic-loader/package.json +++ b/packages/cspell-dynamic-loader/package.json @@ -38,7 +38,7 @@ "author": "Jason Dent", "license": "MIT", "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "peerDependencies": { "cspell": "^5.8.2", diff --git a/packages/cspell-glob/package.json b/packages/cspell-glob/package.json index da7a6037628..308410b4646 100644 --- a/packages/cspell-glob/package.json +++ b/packages/cspell-glob/package.json @@ -38,7 +38,7 @@ "url": "https://github.com/streetsidesoftware/cspell/labels/cspell-glob" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "dependencies": { "micromatch": "^4.0.4" diff --git a/packages/cspell-io/package-lock.json b/packages/cspell-io/package-lock.json index d9ac21700a5..2ec5105aa42 100644 --- a/packages/cspell-io/package-lock.json +++ b/packages/cspell-io/package-lock.json @@ -1544,11 +1544,12 @@ "dev": true }, "iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, "requires": { - "safer-buffer": ">= 2.1.2 < 3.0.0" + "safer-buffer": ">= 2.1.2 < 3" } }, "import-local": { @@ -1693,11 +1694,6 @@ "istanbul-lib-report": "^3.0.0" } }, - "iterable-to-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/iterable-to-stream/-/iterable-to-stream-2.0.0.tgz", - "integrity": "sha512-efkLePxXjJk92hvN+2rS3tGJTRn8/tqXjmZvPI6LQ29xCj2sUF4zW8hkMsVe3jpTkxtMZ89xsKnz9FaRqNWM6g==" - }, "jest": { "version": "27.1.1", "resolved": "https://registry.npmjs.org/jest/-/jest-27.1.1.tgz", @@ -2666,7 +2662,8 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true }, "saxes": { "version": "5.0.1", @@ -2993,17 +2990,6 @@ "dev": true, "requires": { "iconv-lite": "0.4.24" - }, - "dependencies": { - "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dev": true, - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - } } }, "whatwg-mimetype": { diff --git a/packages/cspell-io/package.json b/packages/cspell-io/package.json index f92fd33643f..5064e37c36e 100644 --- a/packages/cspell-io/package.json +++ b/packages/cspell-io/package.json @@ -33,12 +33,9 @@ "url": "https://github.com/streetsidesoftware/cspell/labels/cspell-io" }, "homepage": "https://github.com/streetsidesoftware/cspell#readme", - "dependencies": { - "iconv-lite": "^0.6.3", - "iterable-to-stream": "^2.0.0" - }, + "dependencies": {}, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@types/fs-extra": "^9.0.12", diff --git a/packages/cspell-io/src/async/asyncIterable.test.ts b/packages/cspell-io/src/async/asyncIterable.test.ts new file mode 100644 index 00000000000..5dd12b6261a --- /dev/null +++ b/packages/cspell-io/src/async/asyncIterable.test.ts @@ -0,0 +1,14 @@ +import { toArray } from './asyncIterable'; + +describe('asyncIterable', () => { + test('toArray', async () => { + const r = toArray(emit([1, 2, 3, Promise.resolve(4), 5])); + await expect(r).resolves.toEqual([1, 2, 3, 4, 5]); + }); +}); + +async function* emit(values: Iterable>): AsyncIterableIterator { + for await (const v of values) { + yield v; + } +} diff --git a/packages/cspell-io/src/file/fileReader.test.ts b/packages/cspell-io/src/file/fileReader.test.ts index 2b5d1d48125..84893e762a5 100644 --- a/packages/cspell-io/src/file/fileReader.test.ts +++ b/packages/cspell-io/src/file/fileReader.test.ts @@ -1,90 +1,15 @@ import * as fReader from './fileReader'; import * as fs from 'fs-extra'; -import * as path from 'path'; -import { Readable } from 'stream'; -import * as asyncIterable from '../async/asyncIterable'; describe('Validate the fileReader', () => { - const samplePath = path.join(__dirname, '..', '..', 'samples'); - const fileCities = path.join(samplePath, 'cities.txt'); - const sampleFiles = ['cities.txt', 'cities.CRLF.txt', 'cities.noEOL.txt'].map((f) => path.join(samplePath, f)); - test('tests reading a file', async () => { const expected = await fs.readFile(__filename, 'utf8'); const result = await fReader.readFile(__filename, 'utf8'); expect(result).toBe(expected); }); - test('tests stringsToLines', async () => { - const strings = stringToStream('a1\n2\n3\n4', '5\n6'); - const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); - expect(a).toEqual(['a1', '2', '3', '45', '6']); - }); - - test('tests stringsToLines trailing new line', async () => { - const strings = stringToStream('a1\n2\n3\n4', '5\n6\n'); - const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); - expect(a).toEqual(['a1', '2', '3', '45', '6', '']); - }); - - test('the file reader', async () => { - const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); - const actual = lines.join('\n'); - const expected = fs.readFileSync(__filename, 'utf8'); - expect(actual).toBe(expected); - }); - - test('the lineReaderAsync', async () => { - const lines = await asyncIterable.toArray(fReader.lineReaderAsync(__filename)); - const expected = fs.readFileSync(__filename, 'utf8').split('\n'); - expect(lines).toEqual(expected); - }); - - test('tests reading the cities sample', async () => { - const lines = await asyncIterable.toArray(fReader.lineReaderAsync(fileCities)); - const file = await fs.readFile(fileCities, 'utf8'); - expect(lines).toEqual(file.split('\n')); - }); - - test('tests streamFileLineByLineAsync', async () => { - await Promise.all( - sampleFiles.map(async (filename) => { - const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(filename)); - const file = await fs.readFile(filename, 'utf8'); - // compare to file: ${filename} - expect(lines).toEqual(file.split(/\r?\n/)); - }) - ); - }); - - test('tests streamFileLineByLineAsync 2', async () => { - const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); - const file = await fs.readFile(__filename, 'utf8'); - expect(lines).toEqual(file.split('\n')); - }); - test('missing file', async () => { - const result = asyncIterable.toArray(fReader.lineReaderAsync(__filename + 'not.found')); - return result.then( - () => { - expect('not to be here').toBe(true); - return; - }, - (e) => { - // expect(e).to.be.instanceof(Error); // Since jest currently mocks Error, this test fails. - expect(e.code).toBe('ENOENT'); - } - ); + const result = fReader.readFile(__filename + '.missing.file', 'utf8'); + await expect(result).rejects.toEqual(expect.objectContaining({ code: 'ENOENT' })); }); }); - -function stringToStream(...strings: string[]): NodeJS.ReadableStream { - return new Readable({ - read: function () { - for (const s of strings) { - this.push(s); - } - this.push(null); - }, - }); -} diff --git a/packages/cspell-io/src/file/fileReader.ts b/packages/cspell-io/src/file/fileReader.ts index 1f46009a37b..b87a00c2fa8 100644 --- a/packages/cspell-io/src/file/fileReader.ts +++ b/packages/cspell-io/src/file/fileReader.ts @@ -1,9 +1,7 @@ // cSpell:ignore curr // cSpell:words zlib iconv import * as fs from 'fs'; -import * as iconv from 'iconv-lite'; import * as zlib from 'zlib'; -import * as readline from 'readline'; const defaultEncoding: BufferEncoding = 'utf8'; @@ -23,135 +21,14 @@ export function readFile(filename: string, encoding: BufferEncoding = defaultEnc }); } -/** - * Reads a file line by line. The last value emitted by the Observable is always an empty string. - * @param filename - * @param encoding defaults to 'utf8' - */ -export function lineReaderAsync(filename: string, encoding: BufferEncoding = defaultEncoding): AsyncIterable { - return streamFileLineByLineAsync(filename, encoding); -} - -function prepareFileStream(filename: string, encoding: string, fnError: (e: Error) => void) { +function prepareFileStream(filename: string, encoding: BufferEncoding, fnError: (e: Error) => void) { const pipes: NodeJS.ReadWriteStream[] = []; if (filename.match(/\.gz$/i)) { pipes.push(zlib.createGunzip()); } - pipes.push(iconv.decodeStream(encoding)); const fileStream = fs.createReadStream(filename); fileStream.on('error', fnError); const stream = pipes.reduce((s, p) => s.pipe(p).on('error', fnError), fileStream); + stream.setEncoding(encoding); return stream; } - -/** - * Emit a file line by line - * @param filename full path to the file to read. - * @param encoding defaults to 'utf8' - */ -export function streamFileLineByLineAsync( - filename: string, - encoding: BufferEncoding = defaultEncoding -): AsyncIterableIterator { - const fnError = (e: Error) => { - iter.throw && iter.throw(e); - }; - const stream = prepareFileStream(filename, encoding, fnError); - const iter = streamLineByLineAsync(stream); - return iter; -} - -type Resolve = (value: T | Promise) => void; -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type Reject = (reason?: any) => void; - -interface Resolvers> { - resolve: Resolve; - reject: Reject; -} - -/** - * Emit a file line by line - * @param filename full path to the file to read. - * @param encoding defaults to 'utf8' - */ -export function streamLineByLineAsync( - stream: NodeJS.ReadableStream, - encoding: BufferEncoding = defaultEncoding -): AsyncIterableIterator { - let data = '.'; - let done = false; - let error: Error | undefined; - const buffer: string[] = []; - const pending: Resolvers[] = []; - const fnError = (e: Error | undefined) => { - error = e; - }; - const fnComplete = () => { - // readline will consume the last newline without emitting an empty last line. - // If the last data read contains a new line, then emit an empty string. - if (data.match(/(?:(?:\r?\n)|(?:\r))$/)) { - buffer.push(''); - } - processBuffer(); - done = true; - }; - // We want to capture the last line. - stream.on('data', (d) => (data = dataToString(d, encoding))); - stream.on('error', fnError); - const rl = readline.createInterface({ - input: stream, - terminal: false, - }); - rl.on('close', fnComplete); - rl.on('line', (text: string) => { - buffer.push(text); - processBuffer(); - }); - - function registerPromise(resolve: Resolve>, reject: Reject) { - pending.push({ resolve, reject }); - processBuffer(); - } - - function processBuffer() { - if (error && pending.length && !buffer.length) { - const p = pending.shift(); - p?.reject(error); - return; - } - while (pending.length && buffer.length) { - const p = pending.shift(); - const b = buffer.shift(); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - p?.resolve({ done: false, value: b! }); - } - if (!done) { - pending.length ? rl.resume() : rl.pause(); - } - if (done && pending.length && !buffer.length) { - const p = pending.shift(); - p?.resolve({ done, value: undefined }); - } - } - - const iter: AsyncIterableIterator = { - [Symbol.asyncIterator]: () => iter, - next() { - return new Promise(registerPromise); - }, - throw(e) { - fnError(e); - return new Promise(registerPromise); - }, - }; - - return iter; -} - -function dataToString(data: string | Buffer, encoding: BufferEncoding = 'utf8'): string { - if (typeof data === 'string') { - return data; - } - return data.toString(encoding); -} diff --git a/packages/cspell-io/src/file/fileWriter.ts b/packages/cspell-io/src/file/fileWriter.ts index a3e9896385d..2a75bfd8ba2 100644 --- a/packages/cspell-io/src/file/fileWriter.ts +++ b/packages/cspell-io/src/file/fileWriter.ts @@ -1,14 +1,13 @@ import * as fs from 'fs'; import * as zlib from 'zlib'; import * as stream from 'stream'; -import { iterableToStream } from 'iterable-to-stream'; export function writeToFile(filename: string, data: string): NodeJS.WritableStream { return writeToFileIterable(filename, [data]); } export function writeToFileIterable(filename: string, data: Iterable): NodeJS.WritableStream { - const sourceStream = iterableToStream(data); + const sourceStream = stream.Readable.from(data); const writeStream = fs.createWriteStream(filename); const zip = filename.match(/\.gz$/) ? zlib.createGzip() : new stream.PassThrough(); return sourceStream.pipe(zip).pipe(writeStream); diff --git a/packages/cspell-io/src/index.test.ts b/packages/cspell-io/src/index.test.ts new file mode 100644 index 00000000000..ba2c136551d --- /dev/null +++ b/packages/cspell-io/src/index.test.ts @@ -0,0 +1,7 @@ +import * as index from './index'; + +describe('index', () => { + test('exports', () => { + expect(index.readFile).toBeDefined(); + }); +}); diff --git a/packages/cspell-lib/package.json b/packages/cspell-lib/package.json index 01739c4ebab..7768580c26c 100644 --- a/packages/cspell-lib/package.json +++ b/packages/cspell-lib/package.json @@ -64,7 +64,7 @@ "vscode-uri": "^3.0.2" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@cspell/dict-cpp": "^1.1.40", diff --git a/packages/cspell-tools/package.json b/packages/cspell-tools/package.json index 2713a9b0206..5178f6c4a21 100644 --- a/packages/cspell-tools/package.json +++ b/packages/cspell-tools/package.json @@ -53,7 +53,7 @@ "hunspell-reader": "^5.9.0" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@types/fs-extra": "^9.0.12", diff --git a/packages/cspell-trie-lib/package.json b/packages/cspell-trie-lib/package.json index 202ec9ec429..accfa4adeb3 100644 --- a/packages/cspell-trie-lib/package.json +++ b/packages/cspell-trie-lib/package.json @@ -39,7 +39,7 @@ "gensequence": "^3.1.1" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@cspell/dict-en_us": "^2.1.0", diff --git a/packages/cspell-trie/package-lock.json b/packages/cspell-trie/package-lock.json index 52b505f753b..07b19a14f8b 100644 --- a/packages/cspell-trie/package-lock.json +++ b/packages/cspell-trie/package-lock.json @@ -1705,11 +1705,6 @@ "istanbul-lib-report": "^3.0.0" } }, - "iterable-to-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/iterable-to-stream/-/iterable-to-stream-2.0.0.tgz", - "integrity": "sha512-efkLePxXjJk92hvN+2rS3tGJTRn8/tqXjmZvPI6LQ29xCj2sUF4zW8hkMsVe3jpTkxtMZ89xsKnz9FaRqNWM6g==" - }, "jest": { "version": "27.1.1", "resolved": "https://registry.npmjs.org/jest/-/jest-27.1.1.tgz", diff --git a/packages/cspell-trie/package.json b/packages/cspell-trie/package.json index 1ace741b7b6..36568f98cb8 100644 --- a/packages/cspell-trie/package.json +++ b/packages/cspell-trie/package.json @@ -42,11 +42,10 @@ "commander": "^8.2.0", "cspell-trie-lib": "^5.9.0", "fs-extra": "^10.0.0", - "gensequence": "^3.1.1", - "iterable-to-stream": "^2.0.0" + "gensequence": "^3.1.1" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@types/fs-extra": "^9.0.12", diff --git a/packages/cspell-trie/src/app.ts b/packages/cspell-trie/src/app.ts index 3f4f3c6d22a..8010c72056d 100644 --- a/packages/cspell-trie/src/app.ts +++ b/packages/cspell-trie/src/app.ts @@ -4,7 +4,7 @@ import { mkdirp } from 'fs-extra'; import * as path from 'path'; import * as Trie from 'cspell-trie-lib'; import { Sequence, genSequence } from 'gensequence'; -import { iterableToStream } from 'iterable-to-stream'; +import * as stream from 'stream'; import * as zlib from 'zlib'; const UTF8: BufferEncoding = 'utf8'; @@ -41,7 +41,7 @@ export function run(program: commander.Command, argv: string[]): Promise { serialStream.pipe(outputStream).on('finish', () => resolve()); @@ -61,7 +61,7 @@ export function run(program: commander.Command, argv: string[]): Promise = Trie.iteratorTrieWords(root); const outputStream = await pOutputStream; return new Promise((resolve) => { - iterableToStream(words.map((a) => a + '\n')) + stream.Readable.from(words.map((a) => a + '\n')) .pipe(outputStream) .on('finish', () => resolve()); }); diff --git a/packages/cspell-trie2-lib/package.json b/packages/cspell-trie2-lib/package.json index 174c6f43b8d..eca7725540f 100644 --- a/packages/cspell-trie2-lib/package.json +++ b/packages/cspell-trie2-lib/package.json @@ -35,7 +35,7 @@ }, "homepage": "https://github.com/streetsidesoftware/cspell#readme", "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "jest": "^27.1.1", diff --git a/packages/cspell-types/package.json b/packages/cspell-types/package.json index 2e05036a727..39b9b1920c0 100644 --- a/packages/cspell-types/package.json +++ b/packages/cspell-types/package.json @@ -44,7 +44,7 @@ "author": "Jason Dent", "license": "MIT", "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "ajv-cli": "^5.0.0", diff --git a/packages/cspell/package.json b/packages/cspell/package.json index d43a80d2938..4253f8784cc 100644 --- a/packages/cspell/package.json +++ b/packages/cspell/package.json @@ -80,7 +80,7 @@ "vscode-uri": "^3.0.2" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@cspell/cspell-types": "^5.9.0", diff --git a/packages/hunspell-reader/package.json b/packages/hunspell-reader/package.json index 5229b843a51..99c4e40491a 100644 --- a/packages/hunspell-reader/package.json +++ b/packages/hunspell-reader/package.json @@ -66,6 +66,6 @@ "rules": {} }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" } } diff --git a/test-packages/test-cspell-io/src/index.ts b/test-packages/test-cspell-io/src/index.ts index 9deda5bd592..648b6307809 100644 --- a/test-packages/test-cspell-io/src/index.ts +++ b/test-packages/test-cspell-io/src/index.ts @@ -6,7 +6,7 @@ console.log('start'); /** * The main goal here is to make sure it compiles. The unit tests are validation that it compiled as expected. */ -const functions = [io.asyncIterableToArray, io.lineReaderAsync, io.readFile]; +const functions = [io.readFile, io.writeToFile]; functions.forEach((fn) => assert(typeof fn === 'function', "typeof %o === 'function'", fn));