From e688e2bfaef3c22270a41cac913551bf14692962 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Sat, 11 Sep 2021 08:56:14 +0200 Subject: [PATCH 1/4] fix: drop need for iconv-lite and iterable-to-stream - iterable-to-stream - was replaced with the now built-in method. - iconv-lite - was never really needed since all files are expected to be in `utf-8`. - Bump the minimum version of not to the original LTS version 12.13.0 --- packages/cspell-bundled-dicts/package.json | 2 +- packages/cspell-dynamic-loader/package.json | 2 +- packages/cspell-glob/package.json | 2 +- packages/cspell-io/package-lock.json | 28 ++++++--------------- packages/cspell-io/package.json | 7 ++---- packages/cspell-io/src/file/fileReader.ts | 5 ++-- packages/cspell-io/src/file/fileWriter.ts | 3 +-- packages/cspell-lib/package.json | 2 +- packages/cspell-tools/package.json | 2 +- packages/cspell-trie-lib/package.json | 2 +- packages/cspell-trie/package-lock.json | 5 ---- packages/cspell-trie/package.json | 5 ++-- packages/cspell-trie/src/app.ts | 6 ++--- packages/cspell-trie2-lib/package.json | 2 +- packages/cspell-types/package.json | 2 +- packages/cspell/package.json | 2 +- packages/hunspell-reader/package.json | 2 +- 17 files changed, 27 insertions(+), 52 deletions(-) diff --git a/packages/cspell-bundled-dicts/package.json b/packages/cspell-bundled-dicts/package.json index 906b3c487a5..e3322260cbb 100644 --- a/packages/cspell-bundled-dicts/package.json +++ b/packages/cspell-bundled-dicts/package.json @@ -80,7 +80,7 @@ "@cspell/dict-typescript": "^1.0.19" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@cspell/cspell-tools": "^5.9.0", diff --git a/packages/cspell-dynamic-loader/package.json b/packages/cspell-dynamic-loader/package.json index 78c876d607d..f0d13e494d5 100644 --- a/packages/cspell-dynamic-loader/package.json +++ b/packages/cspell-dynamic-loader/package.json @@ -38,7 +38,7 @@ "author": "Jason Dent", "license": "MIT", "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "peerDependencies": { "cspell": "^5.8.2", diff --git a/packages/cspell-glob/package.json b/packages/cspell-glob/package.json index da7a6037628..308410b4646 100644 --- a/packages/cspell-glob/package.json +++ b/packages/cspell-glob/package.json @@ -38,7 +38,7 @@ "url": "https://github.com/streetsidesoftware/cspell/labels/cspell-glob" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "dependencies": { "micromatch": "^4.0.4" diff --git a/packages/cspell-io/package-lock.json b/packages/cspell-io/package-lock.json index d9ac21700a5..2ec5105aa42 100644 --- a/packages/cspell-io/package-lock.json +++ b/packages/cspell-io/package-lock.json @@ -1544,11 +1544,12 @@ "dev": true }, "iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, "requires": { - "safer-buffer": ">= 2.1.2 < 3.0.0" + "safer-buffer": ">= 2.1.2 < 3" } }, "import-local": { @@ -1693,11 +1694,6 @@ "istanbul-lib-report": "^3.0.0" } }, - "iterable-to-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/iterable-to-stream/-/iterable-to-stream-2.0.0.tgz", - "integrity": "sha512-efkLePxXjJk92hvN+2rS3tGJTRn8/tqXjmZvPI6LQ29xCj2sUF4zW8hkMsVe3jpTkxtMZ89xsKnz9FaRqNWM6g==" - }, "jest": { "version": "27.1.1", "resolved": "https://registry.npmjs.org/jest/-/jest-27.1.1.tgz", @@ -2666,7 +2662,8 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true }, "saxes": { "version": "5.0.1", @@ -2993,17 +2990,6 @@ "dev": true, "requires": { "iconv-lite": "0.4.24" - }, - "dependencies": { - "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dev": true, - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - } } }, "whatwg-mimetype": { diff --git a/packages/cspell-io/package.json b/packages/cspell-io/package.json index f92fd33643f..5064e37c36e 100644 --- a/packages/cspell-io/package.json +++ b/packages/cspell-io/package.json @@ -33,12 +33,9 @@ "url": "https://github.com/streetsidesoftware/cspell/labels/cspell-io" }, "homepage": "https://github.com/streetsidesoftware/cspell#readme", - "dependencies": { - "iconv-lite": "^0.6.3", - "iterable-to-stream": "^2.0.0" - }, + "dependencies": {}, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@types/fs-extra": "^9.0.12", diff --git a/packages/cspell-io/src/file/fileReader.ts b/packages/cspell-io/src/file/fileReader.ts index 1f46009a37b..817677b783d 100644 --- a/packages/cspell-io/src/file/fileReader.ts +++ b/packages/cspell-io/src/file/fileReader.ts @@ -1,7 +1,6 @@ // cSpell:ignore curr // cSpell:words zlib iconv import * as fs from 'fs'; -import * as iconv from 'iconv-lite'; import * as zlib from 'zlib'; import * as readline from 'readline'; @@ -32,15 +31,15 @@ export function lineReaderAsync(filename: string, encoding: BufferEncoding = def return streamFileLineByLineAsync(filename, encoding); } -function prepareFileStream(filename: string, encoding: string, fnError: (e: Error) => void) { +function prepareFileStream(filename: string, encoding: BufferEncoding, fnError: (e: Error) => void) { const pipes: NodeJS.ReadWriteStream[] = []; if (filename.match(/\.gz$/i)) { pipes.push(zlib.createGunzip()); } - pipes.push(iconv.decodeStream(encoding)); const fileStream = fs.createReadStream(filename); fileStream.on('error', fnError); const stream = pipes.reduce((s, p) => s.pipe(p).on('error', fnError), fileStream); + stream.setEncoding(encoding); return stream; } diff --git a/packages/cspell-io/src/file/fileWriter.ts b/packages/cspell-io/src/file/fileWriter.ts index a3e9896385d..2a75bfd8ba2 100644 --- a/packages/cspell-io/src/file/fileWriter.ts +++ b/packages/cspell-io/src/file/fileWriter.ts @@ -1,14 +1,13 @@ import * as fs from 'fs'; import * as zlib from 'zlib'; import * as stream from 'stream'; -import { iterableToStream } from 'iterable-to-stream'; export function writeToFile(filename: string, data: string): NodeJS.WritableStream { return writeToFileIterable(filename, [data]); } export function writeToFileIterable(filename: string, data: Iterable): NodeJS.WritableStream { - const sourceStream = iterableToStream(data); + const sourceStream = stream.Readable.from(data); const writeStream = fs.createWriteStream(filename); const zip = filename.match(/\.gz$/) ? zlib.createGzip() : new stream.PassThrough(); return sourceStream.pipe(zip).pipe(writeStream); diff --git a/packages/cspell-lib/package.json b/packages/cspell-lib/package.json index 01739c4ebab..7768580c26c 100644 --- a/packages/cspell-lib/package.json +++ b/packages/cspell-lib/package.json @@ -64,7 +64,7 @@ "vscode-uri": "^3.0.2" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@cspell/dict-cpp": "^1.1.40", diff --git a/packages/cspell-tools/package.json b/packages/cspell-tools/package.json index 2713a9b0206..5178f6c4a21 100644 --- a/packages/cspell-tools/package.json +++ b/packages/cspell-tools/package.json @@ -53,7 +53,7 @@ "hunspell-reader": "^5.9.0" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@types/fs-extra": "^9.0.12", diff --git a/packages/cspell-trie-lib/package.json b/packages/cspell-trie-lib/package.json index 202ec9ec429..accfa4adeb3 100644 --- a/packages/cspell-trie-lib/package.json +++ b/packages/cspell-trie-lib/package.json @@ -39,7 +39,7 @@ "gensequence": "^3.1.1" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@cspell/dict-en_us": "^2.1.0", diff --git a/packages/cspell-trie/package-lock.json b/packages/cspell-trie/package-lock.json index 52b505f753b..07b19a14f8b 100644 --- a/packages/cspell-trie/package-lock.json +++ b/packages/cspell-trie/package-lock.json @@ -1705,11 +1705,6 @@ "istanbul-lib-report": "^3.0.0" } }, - "iterable-to-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/iterable-to-stream/-/iterable-to-stream-2.0.0.tgz", - "integrity": "sha512-efkLePxXjJk92hvN+2rS3tGJTRn8/tqXjmZvPI6LQ29xCj2sUF4zW8hkMsVe3jpTkxtMZ89xsKnz9FaRqNWM6g==" - }, "jest": { "version": "27.1.1", "resolved": "https://registry.npmjs.org/jest/-/jest-27.1.1.tgz", diff --git a/packages/cspell-trie/package.json b/packages/cspell-trie/package.json index 1ace741b7b6..36568f98cb8 100644 --- a/packages/cspell-trie/package.json +++ b/packages/cspell-trie/package.json @@ -42,11 +42,10 @@ "commander": "^8.2.0", "cspell-trie-lib": "^5.9.0", "fs-extra": "^10.0.0", - "gensequence": "^3.1.1", - "iterable-to-stream": "^2.0.0" + "gensequence": "^3.1.1" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@types/fs-extra": "^9.0.12", diff --git a/packages/cspell-trie/src/app.ts b/packages/cspell-trie/src/app.ts index 3f4f3c6d22a..8010c72056d 100644 --- a/packages/cspell-trie/src/app.ts +++ b/packages/cspell-trie/src/app.ts @@ -4,7 +4,7 @@ import { mkdirp } from 'fs-extra'; import * as path from 'path'; import * as Trie from 'cspell-trie-lib'; import { Sequence, genSequence } from 'gensequence'; -import { iterableToStream } from 'iterable-to-stream'; +import * as stream from 'stream'; import * as zlib from 'zlib'; const UTF8: BufferEncoding = 'utf8'; @@ -41,7 +41,7 @@ export function run(program: commander.Command, argv: string[]): Promise { serialStream.pipe(outputStream).on('finish', () => resolve()); @@ -61,7 +61,7 @@ export function run(program: commander.Command, argv: string[]): Promise = Trie.iteratorTrieWords(root); const outputStream = await pOutputStream; return new Promise((resolve) => { - iterableToStream(words.map((a) => a + '\n')) + stream.Readable.from(words.map((a) => a + '\n')) .pipe(outputStream) .on('finish', () => resolve()); }); diff --git a/packages/cspell-trie2-lib/package.json b/packages/cspell-trie2-lib/package.json index 174c6f43b8d..eca7725540f 100644 --- a/packages/cspell-trie2-lib/package.json +++ b/packages/cspell-trie2-lib/package.json @@ -35,7 +35,7 @@ }, "homepage": "https://github.com/streetsidesoftware/cspell#readme", "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "jest": "^27.1.1", diff --git a/packages/cspell-types/package.json b/packages/cspell-types/package.json index 2e05036a727..39b9b1920c0 100644 --- a/packages/cspell-types/package.json +++ b/packages/cspell-types/package.json @@ -44,7 +44,7 @@ "author": "Jason Dent", "license": "MIT", "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "ajv-cli": "^5.0.0", diff --git a/packages/cspell/package.json b/packages/cspell/package.json index d43a80d2938..4253f8784cc 100644 --- a/packages/cspell/package.json +++ b/packages/cspell/package.json @@ -80,7 +80,7 @@ "vscode-uri": "^3.0.2" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" }, "devDependencies": { "@cspell/cspell-types": "^5.9.0", diff --git a/packages/hunspell-reader/package.json b/packages/hunspell-reader/package.json index 5229b843a51..99c4e40491a 100644 --- a/packages/hunspell-reader/package.json +++ b/packages/hunspell-reader/package.json @@ -66,6 +66,6 @@ "rules": {} }, "engines": { - "node": ">=12.0.0" + "node": ">=12.13.0" } } From 8dd73be67f52ce10568f3c250731f593e521942c Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Sat, 11 Sep 2021 16:42:44 +0200 Subject: [PATCH 2/4] feat: Remove async line reader from cspell-io --- packages/cspell-io/.vscode/launch.json | 19 ++- packages/cspell-io/src/async/asyncIterable.ts | 11 -- .../cspell-io/src/file/fileReader.test.ts | 79 +-------- packages/cspell-io/src/file/fileReader.ts | 161 ++---------------- packages/cspell-io/src/file/fileWriter.ts | 12 +- packages/cspell-io/src/index.ts | 1 - test-packages/test-cspell-io/src/index.ts | 2 +- .../test-cspell-lib-webpack/src/index.ts | 2 +- test-packages/test-cspell-lib/src/index.ts | 2 +- 9 files changed, 41 insertions(+), 248 deletions(-) delete mode 100644 packages/cspell-io/src/async/asyncIterable.ts diff --git a/packages/cspell-io/.vscode/launch.json b/packages/cspell-io/.vscode/launch.json index 5a432bdc8ce..d5aea7bce41 100644 --- a/packages/cspell-io/.vscode/launch.json +++ b/packages/cspell-io/.vscode/launch.json @@ -7,12 +7,19 @@ { "type": "node", "request": "launch", - "name": "Launch Program", - "program": "${workspaceFolder}/dist/index.js", - "preLaunchTask": "tsc: build - tsconfig.json", - "outFiles": [ - "${workspaceFolder}/dist/**/*.js" - ] + "name": "cspell-io: Jest current-file", + "program": "${workspaceFolder}/../../node_modules/.bin/jest", + "cwd": "${workspaceFolder}", + "args": [ + "--runInBand", + "${fileBasename}" + ], + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "disableOptimisticBPs": true, + "windows": { + "program": "${workspaceFolder}/../../node_modules/jest/bin/jest", + } } ] } diff --git a/packages/cspell-io/src/async/asyncIterable.ts b/packages/cspell-io/src/async/asyncIterable.ts deleted file mode 100644 index 4bb5e64db8d..00000000000 --- a/packages/cspell-io/src/async/asyncIterable.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Reads an entire iterable and converts it into a promise. - * @param asyncIterable the async iterable to wait for. - */ -export async function toArray(asyncIterable: AsyncIterable | Iterable | Iterable>): Promise { - const data: T[] = []; - for await (const item of asyncIterable) { - data.push(item); - } - return data; -} diff --git a/packages/cspell-io/src/file/fileReader.test.ts b/packages/cspell-io/src/file/fileReader.test.ts index 2b5d1d48125..45829fffe18 100644 --- a/packages/cspell-io/src/file/fileReader.test.ts +++ b/packages/cspell-io/src/file/fileReader.test.ts @@ -1,90 +1,15 @@ import * as fReader from './fileReader'; import * as fs from 'fs-extra'; -import * as path from 'path'; -import { Readable } from 'stream'; -import * as asyncIterable from '../async/asyncIterable'; describe('Validate the fileReader', () => { - const samplePath = path.join(__dirname, '..', '..', 'samples'); - const fileCities = path.join(samplePath, 'cities.txt'); - const sampleFiles = ['cities.txt', 'cities.CRLF.txt', 'cities.noEOL.txt'].map((f) => path.join(samplePath, f)); - test('tests reading a file', async () => { const expected = await fs.readFile(__filename, 'utf8'); const result = await fReader.readFile(__filename, 'utf8'); expect(result).toBe(expected); }); - test('tests stringsToLines', async () => { - const strings = stringToStream('a1\n2\n3\n4', '5\n6'); - const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); - expect(a).toEqual(['a1', '2', '3', '45', '6']); - }); - - test('tests stringsToLines trailing new line', async () => { - const strings = stringToStream('a1\n2\n3\n4', '5\n6\n'); - const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); - expect(a).toEqual(['a1', '2', '3', '45', '6', '']); - }); - - test('the file reader', async () => { - const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); - const actual = lines.join('\n'); - const expected = fs.readFileSync(__filename, 'utf8'); - expect(actual).toBe(expected); - }); - - test('the lineReaderAsync', async () => { - const lines = await asyncIterable.toArray(fReader.lineReaderAsync(__filename)); - const expected = fs.readFileSync(__filename, 'utf8').split('\n'); - expect(lines).toEqual(expected); - }); - - test('tests reading the cities sample', async () => { - const lines = await asyncIterable.toArray(fReader.lineReaderAsync(fileCities)); - const file = await fs.readFile(fileCities, 'utf8'); - expect(lines).toEqual(file.split('\n')); - }); - - test('tests streamFileLineByLineAsync', async () => { - await Promise.all( - sampleFiles.map(async (filename) => { - const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(filename)); - const file = await fs.readFile(filename, 'utf8'); - // compare to file: ${filename} - expect(lines).toEqual(file.split(/\r?\n/)); - }) - ); - }); - - test('tests streamFileLineByLineAsync 2', async () => { - const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); - const file = await fs.readFile(__filename, 'utf8'); - expect(lines).toEqual(file.split('\n')); - }); - test('missing file', async () => { - const result = asyncIterable.toArray(fReader.lineReaderAsync(__filename + 'not.found')); - return result.then( - () => { - expect('not to be here').toBe(true); - return; - }, - (e) => { - // expect(e).to.be.instanceof(Error); // Since jest currently mocks Error, this test fails. - expect(e.code).toBe('ENOENT'); - } - ); + const result = fReader.readFile(__filename + '.missing.txt', 'utf8'); + await expect(result).rejects.toEqual(expect.objectContaining({ code: 'ENOENT' })); }); }); - -function stringToStream(...strings: string[]): NodeJS.ReadableStream { - return new Readable({ - read: function () { - for (const s of strings) { - this.push(s); - } - this.push(null); - }, - }); -} diff --git a/packages/cspell-io/src/file/fileReader.ts b/packages/cspell-io/src/file/fileReader.ts index 817677b783d..cbd9d7fc56e 100644 --- a/packages/cspell-io/src/file/fileReader.ts +++ b/packages/cspell-io/src/file/fileReader.ts @@ -1,156 +1,27 @@ -// cSpell:ignore curr -// cSpell:words zlib iconv import * as fs from 'fs'; import * as zlib from 'zlib'; -import * as readline from 'readline'; +import { PassThrough, pipeline as pipelineCB } from 'stream'; +import { promisify } from 'util'; -const defaultEncoding: BufferEncoding = 'utf8'; - -export function readFile(filename: string, encoding: BufferEncoding = defaultEncoding): Promise { - return new Promise((resolve, reject) => { - const data: string[] = []; - const stream = prepareFileStream(filename, encoding, reject); - let resolved = false; - function complete() { - resolve(data.join('')); - resolved = resolved || (resolve(data.join('')), true); - } - stream.on('error', reject); - stream.on('data', (d: string) => data.push(d)); - stream.on('close', complete); - stream.on('end', complete); - }); -} +const pipeline = promisify(pipelineCB); -/** - * Reads a file line by line. The last value emitted by the Observable is always an empty string. - * @param filename - * @param encoding defaults to 'utf8' - */ -export function lineReaderAsync(filename: string, encoding: BufferEncoding = defaultEncoding): AsyncIterable { - return streamFileLineByLineAsync(filename, encoding); -} +const defaultEncoding: BufferEncoding = 'utf8'; -function prepareFileStream(filename: string, encoding: BufferEncoding, fnError: (e: Error) => void) { - const pipes: NodeJS.ReadWriteStream[] = []; - if (filename.match(/\.gz$/i)) { - pipes.push(zlib.createGunzip()); - } +export async function readFile(filename: string, encoding: BufferEncoding = defaultEncoding): Promise { + const isGzip = filename.match(/\.gz$/i); const fileStream = fs.createReadStream(filename); - fileStream.on('error', fnError); - const stream = pipes.reduce((s, p) => s.pipe(p).on('error', fnError), fileStream); - stream.setEncoding(encoding); - return stream; -} - -/** - * Emit a file line by line - * @param filename full path to the file to read. - * @param encoding defaults to 'utf8' - */ -export function streamFileLineByLineAsync( - filename: string, - encoding: BufferEncoding = defaultEncoding -): AsyncIterableIterator { - const fnError = (e: Error) => { - iter.throw && iter.throw(e); - }; - const stream = prepareFileStream(filename, encoding, fnError); - const iter = streamLineByLineAsync(stream); - return iter; -} - -type Resolve = (value: T | Promise) => void; -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type Reject = (reason?: any) => void; - -interface Resolvers> { - resolve: Resolve; - reject: Reject; + const zip = isGzip ? zlib.createGunzip() : new PassThrough(); + const t = pipeline(fileStream, zip, streamToText(encoding)); + return await t; } -/** - * Emit a file line by line - * @param filename full path to the file to read. - * @param encoding defaults to 'utf8' - */ -export function streamLineByLineAsync( - stream: NodeJS.ReadableStream, - encoding: BufferEncoding = defaultEncoding -): AsyncIterableIterator { - let data = '.'; - let done = false; - let error: Error | undefined; - const buffer: string[] = []; - const pending: Resolvers[] = []; - const fnError = (e: Error | undefined) => { - error = e; - }; - const fnComplete = () => { - // readline will consume the last newline without emitting an empty last line. - // If the last data read contains a new line, then emit an empty string. - if (data.match(/(?:(?:\r?\n)|(?:\r))$/)) { - buffer.push(''); - } - processBuffer(); - done = true; - }; - // We want to capture the last line. - stream.on('data', (d) => (data = dataToString(d, encoding))); - stream.on('error', fnError); - const rl = readline.createInterface({ - input: stream, - terminal: false, - }); - rl.on('close', fnComplete); - rl.on('line', (text: string) => { - buffer.push(text); - processBuffer(); - }); - - function registerPromise(resolve: Resolve>, reject: Reject) { - pending.push({ resolve, reject }); - processBuffer(); - } - - function processBuffer() { - if (error && pending.length && !buffer.length) { - const p = pending.shift(); - p?.reject(error); - return; - } - while (pending.length && buffer.length) { - const p = pending.shift(); - const b = buffer.shift(); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - p?.resolve({ done: false, value: b! }); +function streamToText(encoding: BufferEncoding): (source: fs.ReadStream) => Promise { + return async function (source: fs.ReadStream): Promise { + const chunks: string[] = []; + source.setEncoding(encoding); // Work with strings rather than `Buffer`s. + for await (const chunk of source) { + chunks.push(chunk); } - if (!done) { - pending.length ? rl.resume() : rl.pause(); - } - if (done && pending.length && !buffer.length) { - const p = pending.shift(); - p?.resolve({ done, value: undefined }); - } - } - - const iter: AsyncIterableIterator = { - [Symbol.asyncIterator]: () => iter, - next() { - return new Promise(registerPromise); - }, - throw(e) { - fnError(e); - return new Promise(registerPromise); - }, + return chunks.join(''); }; - - return iter; -} - -function dataToString(data: string | Buffer, encoding: BufferEncoding = 'utf8'): string { - if (typeof data === 'string') { - return data; - } - return data.toString(encoding); } diff --git a/packages/cspell-io/src/file/fileWriter.ts b/packages/cspell-io/src/file/fileWriter.ts index 2a75bfd8ba2..9a83da6b02f 100644 --- a/packages/cspell-io/src/file/fileWriter.ts +++ b/packages/cspell-io/src/file/fileWriter.ts @@ -1,6 +1,9 @@ import * as fs from 'fs'; import * as zlib from 'zlib'; import * as stream from 'stream'; +import { promisify } from 'util'; + +const pipeline = promisify(stream.pipeline); export function writeToFile(filename: string, data: string): NodeJS.WritableStream { return writeToFileIterable(filename, [data]); @@ -14,9 +17,8 @@ export function writeToFileIterable(filename: string, data: Iterable): N } export function writeToFileIterableP(filename: string, data: Iterable): Promise { - const stream = writeToFileIterable(filename, data); - return new Promise((resolve, reject) => { - stream.on('finish', () => resolve()); - stream.on('error', (e: Error) => reject(e)); - }); + const sourceStream = stream.Readable.from(data); + const writeStream = fs.createWriteStream(filename); + const zip = filename.match(/\.gz$/) ? zlib.createGzip() : new stream.PassThrough(); + return pipeline(sourceStream, zip, writeStream); } diff --git a/packages/cspell-io/src/index.ts b/packages/cspell-io/src/index.ts index bd6fac42fa0..706b0d22881 100644 --- a/packages/cspell-io/src/index.ts +++ b/packages/cspell-io/src/index.ts @@ -1,2 +1 @@ export * from './file'; -export { toArray as asyncIterableToArray } from './async/asyncIterable'; diff --git a/test-packages/test-cspell-io/src/index.ts b/test-packages/test-cspell-io/src/index.ts index 9deda5bd592..70097696a70 100644 --- a/test-packages/test-cspell-io/src/index.ts +++ b/test-packages/test-cspell-io/src/index.ts @@ -6,7 +6,7 @@ console.log('start'); /** * The main goal here is to make sure it compiles. The unit tests are validation that it compiled as expected. */ -const functions = [io.asyncIterableToArray, io.lineReaderAsync, io.readFile]; +const functions = [io.readFile]; functions.forEach((fn) => assert(typeof fn === 'function', "typeof %o === 'function'", fn)); diff --git a/test-packages/test-cspell-lib-webpack/src/index.ts b/test-packages/test-cspell-lib-webpack/src/index.ts index e1d1b2561cd..979db8ec218 100644 --- a/test-packages/test-cspell-lib-webpack/src/index.ts +++ b/test-packages/test-cspell-lib-webpack/src/index.ts @@ -7,7 +7,7 @@ console.log('start'); /** * The main goal here is to make sure it compiles. The unit tests are validation that it compiled as expected. */ -const functions = [lib.asyncIterableToArray, lib.calcOverrideSettings]; +const functions = [lib.calcOverrideSettings]; functions.forEach((fn) => assert(typeof fn === 'function', "typeof %o === 'function'", fn)); console.log('done'); diff --git a/test-packages/test-cspell-lib/src/index.ts b/test-packages/test-cspell-lib/src/index.ts index 714cfcd885d..55a640252a7 100644 --- a/test-packages/test-cspell-lib/src/index.ts +++ b/test-packages/test-cspell-lib/src/index.ts @@ -6,7 +6,7 @@ console.log('start'); /** * The main goal here is to make sure it compiles. The unit tests are validation that it compiled as expected. */ -const functions = [lib.asyncIterableToArray, lib.calcOverrideSettings]; +const functions = [lib.calcOverrideSettings]; functions.forEach((fn) => assert(typeof fn === 'function', "typeof %o === 'function'", fn)); console.log('done'); From 4213c181c30be0c8e5f2969db48504c60cf785bc Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Sat, 11 Sep 2021 17:06:34 +0200 Subject: [PATCH 3/4] Revert "feat: Remove async line reader from cspell-io" This reverts commit 8dd73be67f52ce10568f3c250731f593e521942c. --- packages/cspell-io/.vscode/launch.json | 19 +-- packages/cspell-io/src/async/asyncIterable.ts | 11 ++ .../cspell-io/src/file/fileReader.test.ts | 79 ++++++++- packages/cspell-io/src/file/fileReader.ts | 161 ++++++++++++++++-- packages/cspell-io/src/file/fileWriter.ts | 12 +- packages/cspell-io/src/index.ts | 1 + test-packages/test-cspell-io/src/index.ts | 2 +- .../test-cspell-lib-webpack/src/index.ts | 2 +- test-packages/test-cspell-lib/src/index.ts | 2 +- 9 files changed, 248 insertions(+), 41 deletions(-) create mode 100644 packages/cspell-io/src/async/asyncIterable.ts diff --git a/packages/cspell-io/.vscode/launch.json b/packages/cspell-io/.vscode/launch.json index d5aea7bce41..5a432bdc8ce 100644 --- a/packages/cspell-io/.vscode/launch.json +++ b/packages/cspell-io/.vscode/launch.json @@ -7,19 +7,12 @@ { "type": "node", "request": "launch", - "name": "cspell-io: Jest current-file", - "program": "${workspaceFolder}/../../node_modules/.bin/jest", - "cwd": "${workspaceFolder}", - "args": [ - "--runInBand", - "${fileBasename}" - ], - "console": "integratedTerminal", - "internalConsoleOptions": "neverOpen", - "disableOptimisticBPs": true, - "windows": { - "program": "${workspaceFolder}/../../node_modules/jest/bin/jest", - } + "name": "Launch Program", + "program": "${workspaceFolder}/dist/index.js", + "preLaunchTask": "tsc: build - tsconfig.json", + "outFiles": [ + "${workspaceFolder}/dist/**/*.js" + ] } ] } diff --git a/packages/cspell-io/src/async/asyncIterable.ts b/packages/cspell-io/src/async/asyncIterable.ts new file mode 100644 index 00000000000..4bb5e64db8d --- /dev/null +++ b/packages/cspell-io/src/async/asyncIterable.ts @@ -0,0 +1,11 @@ +/** + * Reads an entire iterable and converts it into a promise. + * @param asyncIterable the async iterable to wait for. + */ +export async function toArray(asyncIterable: AsyncIterable | Iterable | Iterable>): Promise { + const data: T[] = []; + for await (const item of asyncIterable) { + data.push(item); + } + return data; +} diff --git a/packages/cspell-io/src/file/fileReader.test.ts b/packages/cspell-io/src/file/fileReader.test.ts index 45829fffe18..2b5d1d48125 100644 --- a/packages/cspell-io/src/file/fileReader.test.ts +++ b/packages/cspell-io/src/file/fileReader.test.ts @@ -1,15 +1,90 @@ import * as fReader from './fileReader'; import * as fs from 'fs-extra'; +import * as path from 'path'; +import { Readable } from 'stream'; +import * as asyncIterable from '../async/asyncIterable'; describe('Validate the fileReader', () => { + const samplePath = path.join(__dirname, '..', '..', 'samples'); + const fileCities = path.join(samplePath, 'cities.txt'); + const sampleFiles = ['cities.txt', 'cities.CRLF.txt', 'cities.noEOL.txt'].map((f) => path.join(samplePath, f)); + test('tests reading a file', async () => { const expected = await fs.readFile(__filename, 'utf8'); const result = await fReader.readFile(__filename, 'utf8'); expect(result).toBe(expected); }); + test('tests stringsToLines', async () => { + const strings = stringToStream('a1\n2\n3\n4', '5\n6'); + const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); + expect(a).toEqual(['a1', '2', '3', '45', '6']); + }); + + test('tests stringsToLines trailing new line', async () => { + const strings = stringToStream('a1\n2\n3\n4', '5\n6\n'); + const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); + expect(a).toEqual(['a1', '2', '3', '45', '6', '']); + }); + + test('the file reader', async () => { + const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); + const actual = lines.join('\n'); + const expected = fs.readFileSync(__filename, 'utf8'); + expect(actual).toBe(expected); + }); + + test('the lineReaderAsync', async () => { + const lines = await asyncIterable.toArray(fReader.lineReaderAsync(__filename)); + const expected = fs.readFileSync(__filename, 'utf8').split('\n'); + expect(lines).toEqual(expected); + }); + + test('tests reading the cities sample', async () => { + const lines = await asyncIterable.toArray(fReader.lineReaderAsync(fileCities)); + const file = await fs.readFile(fileCities, 'utf8'); + expect(lines).toEqual(file.split('\n')); + }); + + test('tests streamFileLineByLineAsync', async () => { + await Promise.all( + sampleFiles.map(async (filename) => { + const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(filename)); + const file = await fs.readFile(filename, 'utf8'); + // compare to file: ${filename} + expect(lines).toEqual(file.split(/\r?\n/)); + }) + ); + }); + + test('tests streamFileLineByLineAsync 2', async () => { + const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); + const file = await fs.readFile(__filename, 'utf8'); + expect(lines).toEqual(file.split('\n')); + }); + test('missing file', async () => { - const result = fReader.readFile(__filename + '.missing.txt', 'utf8'); - await expect(result).rejects.toEqual(expect.objectContaining({ code: 'ENOENT' })); + const result = asyncIterable.toArray(fReader.lineReaderAsync(__filename + 'not.found')); + return result.then( + () => { + expect('not to be here').toBe(true); + return; + }, + (e) => { + // expect(e).to.be.instanceof(Error); // Since jest currently mocks Error, this test fails. + expect(e.code).toBe('ENOENT'); + } + ); }); }); + +function stringToStream(...strings: string[]): NodeJS.ReadableStream { + return new Readable({ + read: function () { + for (const s of strings) { + this.push(s); + } + this.push(null); + }, + }); +} diff --git a/packages/cspell-io/src/file/fileReader.ts b/packages/cspell-io/src/file/fileReader.ts index cbd9d7fc56e..817677b783d 100644 --- a/packages/cspell-io/src/file/fileReader.ts +++ b/packages/cspell-io/src/file/fileReader.ts @@ -1,27 +1,156 @@ +// cSpell:ignore curr +// cSpell:words zlib iconv import * as fs from 'fs'; import * as zlib from 'zlib'; -import { PassThrough, pipeline as pipelineCB } from 'stream'; -import { promisify } from 'util'; - -const pipeline = promisify(pipelineCB); +import * as readline from 'readline'; const defaultEncoding: BufferEncoding = 'utf8'; -export async function readFile(filename: string, encoding: BufferEncoding = defaultEncoding): Promise { - const isGzip = filename.match(/\.gz$/i); +export function readFile(filename: string, encoding: BufferEncoding = defaultEncoding): Promise { + return new Promise((resolve, reject) => { + const data: string[] = []; + const stream = prepareFileStream(filename, encoding, reject); + let resolved = false; + function complete() { + resolve(data.join('')); + resolved = resolved || (resolve(data.join('')), true); + } + stream.on('error', reject); + stream.on('data', (d: string) => data.push(d)); + stream.on('close', complete); + stream.on('end', complete); + }); +} + +/** + * Reads a file line by line. The last value emitted by the Observable is always an empty string. + * @param filename + * @param encoding defaults to 'utf8' + */ +export function lineReaderAsync(filename: string, encoding: BufferEncoding = defaultEncoding): AsyncIterable { + return streamFileLineByLineAsync(filename, encoding); +} + +function prepareFileStream(filename: string, encoding: BufferEncoding, fnError: (e: Error) => void) { + const pipes: NodeJS.ReadWriteStream[] = []; + if (filename.match(/\.gz$/i)) { + pipes.push(zlib.createGunzip()); + } const fileStream = fs.createReadStream(filename); - const zip = isGzip ? zlib.createGunzip() : new PassThrough(); - const t = pipeline(fileStream, zip, streamToText(encoding)); - return await t; + fileStream.on('error', fnError); + const stream = pipes.reduce((s, p) => s.pipe(p).on('error', fnError), fileStream); + stream.setEncoding(encoding); + return stream; +} + +/** + * Emit a file line by line + * @param filename full path to the file to read. + * @param encoding defaults to 'utf8' + */ +export function streamFileLineByLineAsync( + filename: string, + encoding: BufferEncoding = defaultEncoding +): AsyncIterableIterator { + const fnError = (e: Error) => { + iter.throw && iter.throw(e); + }; + const stream = prepareFileStream(filename, encoding, fnError); + const iter = streamLineByLineAsync(stream); + return iter; +} + +type Resolve = (value: T | Promise) => void; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type Reject = (reason?: any) => void; + +interface Resolvers> { + resolve: Resolve; + reject: Reject; } -function streamToText(encoding: BufferEncoding): (source: fs.ReadStream) => Promise { - return async function (source: fs.ReadStream): Promise { - const chunks: string[] = []; - source.setEncoding(encoding); // Work with strings rather than `Buffer`s. - for await (const chunk of source) { - chunks.push(chunk); +/** + * Emit a file line by line + * @param filename full path to the file to read. + * @param encoding defaults to 'utf8' + */ +export function streamLineByLineAsync( + stream: NodeJS.ReadableStream, + encoding: BufferEncoding = defaultEncoding +): AsyncIterableIterator { + let data = '.'; + let done = false; + let error: Error | undefined; + const buffer: string[] = []; + const pending: Resolvers[] = []; + const fnError = (e: Error | undefined) => { + error = e; + }; + const fnComplete = () => { + // readline will consume the last newline without emitting an empty last line. + // If the last data read contains a new line, then emit an empty string. + if (data.match(/(?:(?:\r?\n)|(?:\r))$/)) { + buffer.push(''); + } + processBuffer(); + done = true; + }; + // We want to capture the last line. + stream.on('data', (d) => (data = dataToString(d, encoding))); + stream.on('error', fnError); + const rl = readline.createInterface({ + input: stream, + terminal: false, + }); + rl.on('close', fnComplete); + rl.on('line', (text: string) => { + buffer.push(text); + processBuffer(); + }); + + function registerPromise(resolve: Resolve>, reject: Reject) { + pending.push({ resolve, reject }); + processBuffer(); + } + + function processBuffer() { + if (error && pending.length && !buffer.length) { + const p = pending.shift(); + p?.reject(error); + return; + } + while (pending.length && buffer.length) { + const p = pending.shift(); + const b = buffer.shift(); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + p?.resolve({ done: false, value: b! }); } - return chunks.join(''); + if (!done) { + pending.length ? rl.resume() : rl.pause(); + } + if (done && pending.length && !buffer.length) { + const p = pending.shift(); + p?.resolve({ done, value: undefined }); + } + } + + const iter: AsyncIterableIterator = { + [Symbol.asyncIterator]: () => iter, + next() { + return new Promise(registerPromise); + }, + throw(e) { + fnError(e); + return new Promise(registerPromise); + }, }; + + return iter; +} + +function dataToString(data: string | Buffer, encoding: BufferEncoding = 'utf8'): string { + if (typeof data === 'string') { + return data; + } + return data.toString(encoding); } diff --git a/packages/cspell-io/src/file/fileWriter.ts b/packages/cspell-io/src/file/fileWriter.ts index 9a83da6b02f..2a75bfd8ba2 100644 --- a/packages/cspell-io/src/file/fileWriter.ts +++ b/packages/cspell-io/src/file/fileWriter.ts @@ -1,9 +1,6 @@ import * as fs from 'fs'; import * as zlib from 'zlib'; import * as stream from 'stream'; -import { promisify } from 'util'; - -const pipeline = promisify(stream.pipeline); export function writeToFile(filename: string, data: string): NodeJS.WritableStream { return writeToFileIterable(filename, [data]); @@ -17,8 +14,9 @@ export function writeToFileIterable(filename: string, data: Iterable): N } export function writeToFileIterableP(filename: string, data: Iterable): Promise { - const sourceStream = stream.Readable.from(data); - const writeStream = fs.createWriteStream(filename); - const zip = filename.match(/\.gz$/) ? zlib.createGzip() : new stream.PassThrough(); - return pipeline(sourceStream, zip, writeStream); + const stream = writeToFileIterable(filename, data); + return new Promise((resolve, reject) => { + stream.on('finish', () => resolve()); + stream.on('error', (e: Error) => reject(e)); + }); } diff --git a/packages/cspell-io/src/index.ts b/packages/cspell-io/src/index.ts index 706b0d22881..bd6fac42fa0 100644 --- a/packages/cspell-io/src/index.ts +++ b/packages/cspell-io/src/index.ts @@ -1 +1,2 @@ export * from './file'; +export { toArray as asyncIterableToArray } from './async/asyncIterable'; diff --git a/test-packages/test-cspell-io/src/index.ts b/test-packages/test-cspell-io/src/index.ts index 70097696a70..9deda5bd592 100644 --- a/test-packages/test-cspell-io/src/index.ts +++ b/test-packages/test-cspell-io/src/index.ts @@ -6,7 +6,7 @@ console.log('start'); /** * The main goal here is to make sure it compiles. The unit tests are validation that it compiled as expected. */ -const functions = [io.readFile]; +const functions = [io.asyncIterableToArray, io.lineReaderAsync, io.readFile]; functions.forEach((fn) => assert(typeof fn === 'function', "typeof %o === 'function'", fn)); diff --git a/test-packages/test-cspell-lib-webpack/src/index.ts b/test-packages/test-cspell-lib-webpack/src/index.ts index 979db8ec218..e1d1b2561cd 100644 --- a/test-packages/test-cspell-lib-webpack/src/index.ts +++ b/test-packages/test-cspell-lib-webpack/src/index.ts @@ -7,7 +7,7 @@ console.log('start'); /** * The main goal here is to make sure it compiles. The unit tests are validation that it compiled as expected. */ -const functions = [lib.calcOverrideSettings]; +const functions = [lib.asyncIterableToArray, lib.calcOverrideSettings]; functions.forEach((fn) => assert(typeof fn === 'function', "typeof %o === 'function'", fn)); console.log('done'); diff --git a/test-packages/test-cspell-lib/src/index.ts b/test-packages/test-cspell-lib/src/index.ts index 55a640252a7..714cfcd885d 100644 --- a/test-packages/test-cspell-lib/src/index.ts +++ b/test-packages/test-cspell-lib/src/index.ts @@ -6,7 +6,7 @@ console.log('start'); /** * The main goal here is to make sure it compiles. The unit tests are validation that it compiled as expected. */ -const functions = [lib.calcOverrideSettings]; +const functions = [lib.asyncIterableToArray, lib.calcOverrideSettings]; functions.forEach((fn) => assert(typeof fn === 'function', "typeof %o === 'function'", fn)); console.log('done'); From 2a1e8f07b536cd7bad05b00a4acd376c240ce990 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Sat, 11 Sep 2021 17:31:34 +0200 Subject: [PATCH 4/4] feat: drop support for `lineReaderAsync` and related functions --- .../cspell-io/src/async/asyncIterable.test.ts | 14 ++ .../cspell-io/src/file/fileReader.test.ts | 79 +----------- packages/cspell-io/src/file/fileReader.ts | 122 ------------------ packages/cspell-io/src/index.test.ts | 7 + test-packages/test-cspell-io/src/index.ts | 2 +- 5 files changed, 24 insertions(+), 200 deletions(-) create mode 100644 packages/cspell-io/src/async/asyncIterable.test.ts create mode 100644 packages/cspell-io/src/index.test.ts diff --git a/packages/cspell-io/src/async/asyncIterable.test.ts b/packages/cspell-io/src/async/asyncIterable.test.ts new file mode 100644 index 00000000000..5dd12b6261a --- /dev/null +++ b/packages/cspell-io/src/async/asyncIterable.test.ts @@ -0,0 +1,14 @@ +import { toArray } from './asyncIterable'; + +describe('asyncIterable', () => { + test('toArray', async () => { + const r = toArray(emit([1, 2, 3, Promise.resolve(4), 5])); + await expect(r).resolves.toEqual([1, 2, 3, 4, 5]); + }); +}); + +async function* emit(values: Iterable>): AsyncIterableIterator { + for await (const v of values) { + yield v; + } +} diff --git a/packages/cspell-io/src/file/fileReader.test.ts b/packages/cspell-io/src/file/fileReader.test.ts index 2b5d1d48125..84893e762a5 100644 --- a/packages/cspell-io/src/file/fileReader.test.ts +++ b/packages/cspell-io/src/file/fileReader.test.ts @@ -1,90 +1,15 @@ import * as fReader from './fileReader'; import * as fs from 'fs-extra'; -import * as path from 'path'; -import { Readable } from 'stream'; -import * as asyncIterable from '../async/asyncIterable'; describe('Validate the fileReader', () => { - const samplePath = path.join(__dirname, '..', '..', 'samples'); - const fileCities = path.join(samplePath, 'cities.txt'); - const sampleFiles = ['cities.txt', 'cities.CRLF.txt', 'cities.noEOL.txt'].map((f) => path.join(samplePath, f)); - test('tests reading a file', async () => { const expected = await fs.readFile(__filename, 'utf8'); const result = await fReader.readFile(__filename, 'utf8'); expect(result).toBe(expected); }); - test('tests stringsToLines', async () => { - const strings = stringToStream('a1\n2\n3\n4', '5\n6'); - const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); - expect(a).toEqual(['a1', '2', '3', '45', '6']); - }); - - test('tests stringsToLines trailing new line', async () => { - const strings = stringToStream('a1\n2\n3\n4', '5\n6\n'); - const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); - expect(a).toEqual(['a1', '2', '3', '45', '6', '']); - }); - - test('the file reader', async () => { - const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); - const actual = lines.join('\n'); - const expected = fs.readFileSync(__filename, 'utf8'); - expect(actual).toBe(expected); - }); - - test('the lineReaderAsync', async () => { - const lines = await asyncIterable.toArray(fReader.lineReaderAsync(__filename)); - const expected = fs.readFileSync(__filename, 'utf8').split('\n'); - expect(lines).toEqual(expected); - }); - - test('tests reading the cities sample', async () => { - const lines = await asyncIterable.toArray(fReader.lineReaderAsync(fileCities)); - const file = await fs.readFile(fileCities, 'utf8'); - expect(lines).toEqual(file.split('\n')); - }); - - test('tests streamFileLineByLineAsync', async () => { - await Promise.all( - sampleFiles.map(async (filename) => { - const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(filename)); - const file = await fs.readFile(filename, 'utf8'); - // compare to file: ${filename} - expect(lines).toEqual(file.split(/\r?\n/)); - }) - ); - }); - - test('tests streamFileLineByLineAsync 2', async () => { - const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); - const file = await fs.readFile(__filename, 'utf8'); - expect(lines).toEqual(file.split('\n')); - }); - test('missing file', async () => { - const result = asyncIterable.toArray(fReader.lineReaderAsync(__filename + 'not.found')); - return result.then( - () => { - expect('not to be here').toBe(true); - return; - }, - (e) => { - // expect(e).to.be.instanceof(Error); // Since jest currently mocks Error, this test fails. - expect(e.code).toBe('ENOENT'); - } - ); + const result = fReader.readFile(__filename + '.missing.file', 'utf8'); + await expect(result).rejects.toEqual(expect.objectContaining({ code: 'ENOENT' })); }); }); - -function stringToStream(...strings: string[]): NodeJS.ReadableStream { - return new Readable({ - read: function () { - for (const s of strings) { - this.push(s); - } - this.push(null); - }, - }); -} diff --git a/packages/cspell-io/src/file/fileReader.ts b/packages/cspell-io/src/file/fileReader.ts index 817677b783d..b87a00c2fa8 100644 --- a/packages/cspell-io/src/file/fileReader.ts +++ b/packages/cspell-io/src/file/fileReader.ts @@ -2,7 +2,6 @@ // cSpell:words zlib iconv import * as fs from 'fs'; import * as zlib from 'zlib'; -import * as readline from 'readline'; const defaultEncoding: BufferEncoding = 'utf8'; @@ -22,15 +21,6 @@ export function readFile(filename: string, encoding: BufferEncoding = defaultEnc }); } -/** - * Reads a file line by line. The last value emitted by the Observable is always an empty string. - * @param filename - * @param encoding defaults to 'utf8' - */ -export function lineReaderAsync(filename: string, encoding: BufferEncoding = defaultEncoding): AsyncIterable { - return streamFileLineByLineAsync(filename, encoding); -} - function prepareFileStream(filename: string, encoding: BufferEncoding, fnError: (e: Error) => void) { const pipes: NodeJS.ReadWriteStream[] = []; if (filename.match(/\.gz$/i)) { @@ -42,115 +32,3 @@ function prepareFileStream(filename: string, encoding: BufferEncoding, fnError: stream.setEncoding(encoding); return stream; } - -/** - * Emit a file line by line - * @param filename full path to the file to read. - * @param encoding defaults to 'utf8' - */ -export function streamFileLineByLineAsync( - filename: string, - encoding: BufferEncoding = defaultEncoding -): AsyncIterableIterator { - const fnError = (e: Error) => { - iter.throw && iter.throw(e); - }; - const stream = prepareFileStream(filename, encoding, fnError); - const iter = streamLineByLineAsync(stream); - return iter; -} - -type Resolve = (value: T | Promise) => void; -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type Reject = (reason?: any) => void; - -interface Resolvers> { - resolve: Resolve; - reject: Reject; -} - -/** - * Emit a file line by line - * @param filename full path to the file to read. - * @param encoding defaults to 'utf8' - */ -export function streamLineByLineAsync( - stream: NodeJS.ReadableStream, - encoding: BufferEncoding = defaultEncoding -): AsyncIterableIterator { - let data = '.'; - let done = false; - let error: Error | undefined; - const buffer: string[] = []; - const pending: Resolvers[] = []; - const fnError = (e: Error | undefined) => { - error = e; - }; - const fnComplete = () => { - // readline will consume the last newline without emitting an empty last line. - // If the last data read contains a new line, then emit an empty string. - if (data.match(/(?:(?:\r?\n)|(?:\r))$/)) { - buffer.push(''); - } - processBuffer(); - done = true; - }; - // We want to capture the last line. - stream.on('data', (d) => (data = dataToString(d, encoding))); - stream.on('error', fnError); - const rl = readline.createInterface({ - input: stream, - terminal: false, - }); - rl.on('close', fnComplete); - rl.on('line', (text: string) => { - buffer.push(text); - processBuffer(); - }); - - function registerPromise(resolve: Resolve>, reject: Reject) { - pending.push({ resolve, reject }); - processBuffer(); - } - - function processBuffer() { - if (error && pending.length && !buffer.length) { - const p = pending.shift(); - p?.reject(error); - return; - } - while (pending.length && buffer.length) { - const p = pending.shift(); - const b = buffer.shift(); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - p?.resolve({ done: false, value: b! }); - } - if (!done) { - pending.length ? rl.resume() : rl.pause(); - } - if (done && pending.length && !buffer.length) { - const p = pending.shift(); - p?.resolve({ done, value: undefined }); - } - } - - const iter: AsyncIterableIterator = { - [Symbol.asyncIterator]: () => iter, - next() { - return new Promise(registerPromise); - }, - throw(e) { - fnError(e); - return new Promise(registerPromise); - }, - }; - - return iter; -} - -function dataToString(data: string | Buffer, encoding: BufferEncoding = 'utf8'): string { - if (typeof data === 'string') { - return data; - } - return data.toString(encoding); -} diff --git a/packages/cspell-io/src/index.test.ts b/packages/cspell-io/src/index.test.ts new file mode 100644 index 00000000000..ba2c136551d --- /dev/null +++ b/packages/cspell-io/src/index.test.ts @@ -0,0 +1,7 @@ +import * as index from './index'; + +describe('index', () => { + test('exports', () => { + expect(index.readFile).toBeDefined(); + }); +}); diff --git a/test-packages/test-cspell-io/src/index.ts b/test-packages/test-cspell-io/src/index.ts index 9deda5bd592..648b6307809 100644 --- a/test-packages/test-cspell-io/src/index.ts +++ b/test-packages/test-cspell-io/src/index.ts @@ -6,7 +6,7 @@ console.log('start'); /** * The main goal here is to make sure it compiles. The unit tests are validation that it compiled as expected. */ -const functions = [io.asyncIterableToArray, io.lineReaderAsync, io.readFile]; +const functions = [io.readFile, io.writeToFile]; functions.forEach((fn) => assert(typeof fn === 'function', "typeof %o === 'function'", fn));