fix: drop need for iconv-lite and iterable-to-stream (#1677)

- iterable-to-stream - was replaced with the now built-in method. - iconv-lite - was never really needed since all files are expected to be in `utf-8`. - Bump the minimum version of not to the original LTS version 12.13.0 - cspell-io - remove unused async line reader and other unused async functions. NOTE: This might be a breaking change for any external party that depends upon cspell-io.
streetsidesoftware · Sep 11, 2021 · c7ffcc7 · c7ffcc7
1 parent ae91c44
commit c7ffcc7
Show file tree

Hide file tree

Showing 21 changed files with 51 additions and 252 deletions.
diff --git a/packages/cspell-bundled-dicts/package.json b/packages/cspell-bundled-dicts/package.json
@@ -80,7 +80,7 @@
     "@cspell/dict-typescript": "^1.0.19"
   },
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=12.13.0"
   },
   "devDependencies": {
     "@cspell/cspell-tools": "^5.9.0",

diff --git a/packages/cspell-dynamic-loader/package.json b/packages/cspell-dynamic-loader/package.json
@@ -38,7 +38,7 @@
   "author": "Jason Dent",
   "license": "MIT",
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=12.13.0"
   },
   "peerDependencies": {
     "cspell": "^5.8.2",

diff --git a/packages/cspell-glob/package.json b/packages/cspell-glob/package.json
@@ -38,7 +38,7 @@
     "url": "https://github.com/streetsidesoftware/cspell/labels/cspell-glob"
   },
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=12.13.0"
   },
   "dependencies": {
     "micromatch": "^4.0.4"

diff --git a/packages/cspell-io/package-lock.json b/packages/cspell-io/package-lock.json
diff --git a/packages/cspell-io/package.json b/packages/cspell-io/package.json
@@ -33,12 +33,9 @@
     "url": "https://github.com/streetsidesoftware/cspell/labels/cspell-io"
   },
   "homepage": "https://github.com/streetsidesoftware/cspell#readme",
-  "dependencies": {
-    "iconv-lite": "^0.6.3",
-    "iterable-to-stream": "^2.0.0"
-  },
+  "dependencies": {},
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=12.13.0"
   },
   "devDependencies": {
     "@types/fs-extra": "^9.0.12",

diff --git a/packages/cspell-io/src/async/asyncIterable.test.ts b/packages/cspell-io/src/async/asyncIterable.test.ts
@@ -0,0 +1,14 @@
+import { toArray } from './asyncIterable';
+
+describe('asyncIterable', () => {
+    test('toArray', async () => {
+        const r = toArray(emit([1, 2, 3, Promise.resolve(4), 5]));
+        await expect(r).resolves.toEqual([1, 2, 3, 4, 5]);
+    });
+});
+
+async function* emit<T>(values: Iterable<T | Promise<T>>): AsyncIterableIterator<T> {
+    for await (const v of values) {
+        yield v;
+    }
+}
diff --git a/packages/cspell-io/src/file/fileReader.test.ts b/packages/cspell-io/src/file/fileReader.test.ts
@@ -1,90 +1,15 @@
 import * as fReader from './fileReader';
 import * as fs from 'fs-extra';
-import * as path from 'path';
-import { Readable } from 'stream';
-import * as asyncIterable from '../async/asyncIterable';
 
 describe('Validate the fileReader', () => {
-    const samplePath = path.join(__dirname, '..', '..', 'samples');
-    const fileCities = path.join(samplePath, 'cities.txt');
-    const sampleFiles = ['cities.txt', 'cities.CRLF.txt', 'cities.noEOL.txt'].map((f) => path.join(samplePath, f));
-
     test('tests reading a file', async () => {
         const expected = await fs.readFile(__filename, 'utf8');
         const result = await fReader.readFile(__filename, 'utf8');
         expect(result).toBe(expected);
     });
 
-    test('tests stringsToLines', async () => {
-        const strings = stringToStream('a1\n2\n3\n4', '5\n6');
-        const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings));
-        expect(a).toEqual(['a1', '2', '3', '45', '6']);
-    });
-
-    test('tests stringsToLines trailing new line', async () => {
-        const strings = stringToStream('a1\n2\n3\n4', '5\n6\n');
-        const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings));
-        expect(a).toEqual(['a1', '2', '3', '45', '6', '']);
-    });
-
-    test('the file reader', async () => {
-        const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename));
-        const actual = lines.join('\n');
-        const expected = fs.readFileSync(__filename, 'utf8');
-        expect(actual).toBe(expected);
-    });
-
-    test('the lineReaderAsync', async () => {
-        const lines = await asyncIterable.toArray(fReader.lineReaderAsync(__filename));
-        const expected = fs.readFileSync(__filename, 'utf8').split('\n');
-        expect(lines).toEqual(expected);
-    });
-
-    test('tests reading the cities sample', async () => {
-        const lines = await asyncIterable.toArray(fReader.lineReaderAsync(fileCities));
-        const file = await fs.readFile(fileCities, 'utf8');
-        expect(lines).toEqual(file.split('\n'));
-    });
-
-    test('tests streamFileLineByLineAsync', async () => {
-        await Promise.all(
-            sampleFiles.map(async (filename) => {
-                const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(filename));
-                const file = await fs.readFile(filename, 'utf8');
-                // compare to file: ${filename}
-                expect(lines).toEqual(file.split(/\r?\n/));
-            })
-        );
-    });
-
-    test('tests streamFileLineByLineAsync 2', async () => {
-        const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename));
-        const file = await fs.readFile(__filename, 'utf8');
-        expect(lines).toEqual(file.split('\n'));
-    });
-
     test('missing file', async () => {
-        const result = asyncIterable.toArray(fReader.lineReaderAsync(__filename + 'not.found'));
-        return result.then(
-            () => {
-                expect('not to be here').toBe(true);
-                return;
-            },
-            (e) => {
-                // expect(e).to.be.instanceof(Error); // Since jest currently mocks Error, this test fails.
-                expect(e.code).toBe('ENOENT');
-            }
-        );
+        const result = fReader.readFile(__filename + '.missing.file', 'utf8');
+        await expect(result).rejects.toEqual(expect.objectContaining({ code: 'ENOENT' }));
     });
 });
-
-function stringToStream(...strings: string[]): NodeJS.ReadableStream {
-    return new Readable({
-        read: function () {
-            for (const s of strings) {
-                this.push(s);
-            }
-            this.push(null);
-        },
-    });
-}
diff --git a/packages/cspell-io/src/file/fileReader.ts b/packages/cspell-io/src/file/fileReader.ts
@@ -1,9 +1,7 @@
 // cSpell:ignore curr
 // cSpell:words zlib iconv
 import * as fs from 'fs';
-import * as iconv from 'iconv-lite';
 import * as zlib from 'zlib';
-import * as readline from 'readline';
 
 const defaultEncoding: BufferEncoding = 'utf8';
 
@@ -23,135 +21,14 @@ export function readFile(filename: string, encoding: BufferEncoding = defaultEnc
     });
 }
 
-/**
- * Reads a file line by line. The last value emitted by the Observable is always an empty string.
- * @param filename
- * @param encoding defaults to 'utf8'
- */
-export function lineReaderAsync(filename: string, encoding: BufferEncoding = defaultEncoding): AsyncIterable<string> {
-    return streamFileLineByLineAsync(filename, encoding);
-}
-
-function prepareFileStream(filename: string, encoding: string, fnError: (e: Error) => void) {
+function prepareFileStream(filename: string, encoding: BufferEncoding, fnError: (e: Error) => void) {
     const pipes: NodeJS.ReadWriteStream[] = [];
     if (filename.match(/\.gz$/i)) {
         pipes.push(zlib.createGunzip());
     }
-    pipes.push(iconv.decodeStream(encoding));
     const fileStream = fs.createReadStream(filename);
     fileStream.on('error', fnError);
     const stream = pipes.reduce<NodeJS.ReadableStream>((s, p) => s.pipe(p).on('error', fnError), fileStream);
+    stream.setEncoding(encoding);
     return stream;
 }
-
-/**
- * Emit a file line by line
- * @param filename full path to the file to read.
- * @param encoding defaults to 'utf8'
- */
-export function streamFileLineByLineAsync(
-    filename: string,
-    encoding: BufferEncoding = defaultEncoding
-): AsyncIterableIterator<string> {
-    const fnError = (e: Error) => {
-        iter.throw && iter.throw(e);
-    };
-    const stream = prepareFileStream(filename, encoding, fnError);
-    const iter = streamLineByLineAsync(stream);
-    return iter;
-}
-
-type Resolve<T> = (value: T | Promise<T>) => void;
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-type Reject = (reason?: any) => void;
-
-interface Resolvers<T = IteratorResult<string>> {
-    resolve: Resolve<T>;
-    reject: Reject;
-}
-
-/**
- * Emit a file line by line
- * @param filename full path to the file to read.
- * @param encoding defaults to 'utf8'
- */
-export function streamLineByLineAsync(
-    stream: NodeJS.ReadableStream,
-    encoding: BufferEncoding = defaultEncoding
-): AsyncIterableIterator<string> {
-    let data = '.';
-    let done = false;
-    let error: Error | undefined;
-    const buffer: string[] = [];
-    const pending: Resolvers[] = [];
-    const fnError = (e: Error | undefined) => {
-        error = e;
-    };
-    const fnComplete = () => {
-        // readline will consume the last newline without emitting an empty last line.
-        // If the last data read contains a new line, then emit an empty string.
-        if (data.match(/(?:(?:\r?\n)|(?:\r))$/)) {
-            buffer.push('');
-        }
-        processBuffer();
-        done = true;
-    };
-    // We want to capture the last line.
-    stream.on('data', (d) => (data = dataToString(d, encoding)));
-    stream.on('error', fnError);
-    const rl = readline.createInterface({
-        input: stream,
-        terminal: false,
-    });
-    rl.on('close', fnComplete);
-    rl.on('line', (text: string) => {
-        buffer.push(text);
-        processBuffer();
-    });
-
-    function registerPromise(resolve: Resolve<IteratorResult<string>>, reject: Reject) {
-        pending.push({ resolve, reject });
-        processBuffer();
-    }
-
-    function processBuffer() {
-        if (error && pending.length && !buffer.length) {
-            const p = pending.shift();
-            p?.reject(error);
-            return;
-        }
-        while (pending.length && buffer.length) {
-            const p = pending.shift();
-            const b = buffer.shift();
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            p?.resolve({ done: false, value: b! });
-        }
-        if (!done) {
-            pending.length ? rl.resume() : rl.pause();
-        }
-        if (done && pending.length && !buffer.length) {
-            const p = pending.shift();
-            p?.resolve({ done, value: undefined });
-        }
-    }
-
-    const iter: AsyncIterableIterator<string> = {
-        [Symbol.asyncIterator]: () => iter,
-        next() {
-            return new Promise(registerPromise);
-        },
-        throw(e) {
-            fnError(e);
-            return new Promise(registerPromise);
-        },
-    };
-
-    return iter;
-}
-
-function dataToString(data: string | Buffer, encoding: BufferEncoding = 'utf8'): string {
-    if (typeof data === 'string') {
-        return data;
-    }
-    return data.toString(encoding);
-}
diff --git a/packages/cspell-io/src/file/fileWriter.ts b/packages/cspell-io/src/file/fileWriter.ts
@@ -1,14 +1,13 @@
 import * as fs from 'fs';
 import * as zlib from 'zlib';
 import * as stream from 'stream';
-import { iterableToStream } from 'iterable-to-stream';
 
 export function writeToFile(filename: string, data: string): NodeJS.WritableStream {
     return writeToFileIterable(filename, [data]);
 }
 
 export function writeToFileIterable(filename: string, data: Iterable<string>): NodeJS.WritableStream {
-    const sourceStream = iterableToStream(data);
+    const sourceStream = stream.Readable.from(data);
     const writeStream = fs.createWriteStream(filename);
     const zip = filename.match(/\.gz$/) ? zlib.createGzip() : new stream.PassThrough();
     return sourceStream.pipe(zip).pipe(writeStream);

diff --git a/packages/cspell-io/src/index.test.ts b/packages/cspell-io/src/index.test.ts
@@ -0,0 +1,7 @@
+import * as index from './index';
+
+describe('index', () => {
+    test('exports', () => {
+        expect(index.readFile).toBeDefined();
+    });
+});
diff --git a/packages/cspell-lib/package.json b/packages/cspell-lib/package.json
@@ -64,7 +64,7 @@
     "vscode-uri": "^3.0.2"
   },
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=12.13.0"
   },
   "devDependencies": {
     "@cspell/dict-cpp": "^1.1.40",

diff --git a/packages/cspell-tools/package.json b/packages/cspell-tools/package.json
@@ -53,7 +53,7 @@
     "hunspell-reader": "^5.9.0"
   },
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=12.13.0"
   },
   "devDependencies": {
     "@types/fs-extra": "^9.0.12",