From 965e4dd4aa71599439a8e5f6c0a527bd7b7145fc Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Tue, 26 Jul 2022 14:10:26 +0200 Subject: [PATCH 1/7] refactor: CSpell-IO to use Service Bus The goal is to have CSpell-IO support both node and web. --- cspell.code-workspace | 3 + packages/cspell-io/package-lock.json | 14 +++++ packages/cspell-io/package.json | 1 + packages/cspell-io/src/CSpellIO.ts | 10 ++++ packages/cspell-io/src/CSpellIONode.test.ts | 37 ++++++++++++ packages/cspell-io/src/CSpellIONode.ts | 25 ++++++++ packages/cspell-io/src/CSpellIOWeb.ts | 25 ++++++++ packages/cspell-io/src/common/stat.test.ts | 27 +++++++++ packages/cspell-io/src/common/stat.ts | 14 +++++ .../src/errors/ErrorNotImplemented.ts | 5 ++ packages/cspell-io/src/file/index.ts | 7 +-- packages/cspell-io/src/handlers/node/file.ts | 58 ++++++++++++++++++ packages/cspell-io/src/index.ts | 13 +++- packages/cspell-io/src/models/Stats.ts | 18 ++++++ packages/cspell-io/src/models/index.ts | 1 + .../src/{ => node}/file/FetchError.test.ts | 0 .../src/{ => node}/file/FetchError.ts | 0 .../src/{ => node}/file/fetch.test.ts | 0 .../cspell-io/src/{ => node}/file/fetch.ts | 0 .../src/{ => node}/file/fileReader.test.ts | 2 +- .../src/{ => node}/file/fileReader.ts | 0 .../src/{ => node}/file/fileWriter.test.ts | 11 ++-- .../src/{ => node}/file/fileWriter.ts | 0 packages/cspell-io/src/node/file/index.ts | 3 + .../src/{ => node}/file/stat.test.ts | 0 .../cspell-io/src/{ => node}/file/stat.ts | 34 ++--------- .../src/{ => node}/file/util.test.ts | 0 .../cspell-io/src/{ => node}/file/util.ts | 8 ++- .../src/requests/RequestFsReadBinaryFile.ts | 7 +++ .../src/requests/RequestFsReadFile.ts | 7 +++ .../src/requests/RequestFsReadFileSync.ts | 7 +++ .../src/requests/RequestZlibInflate.ts | 7 +++ packages/cspell-io/src/requests/index.ts | 3 + .../src/SystemServiceBus.test.ts | 59 +++++++------------ .../src/SystemServiceBus.ts | 57 +++++++----------- .../src/__snapshots__/index.test.ts.snap | 1 + packages/cspell-service-bus/src/bus.test.ts | 34 +++++------ packages/cspell-service-bus/src/bus.ts | 17 +++++- packages/cspell-service-bus/src/index.ts | 11 +++- .../cspell-service-bus/src/request.test.ts | 8 +-- packages/cspell-service-bus/src/request.ts | 20 ++++--- .../cspell-service-bus/src/requestFactory.ts | 19 ++++++ 42 files changed, 425 insertions(+), 148 deletions(-) create mode 100644 packages/cspell-io/src/CSpellIO.ts create mode 100644 packages/cspell-io/src/CSpellIONode.test.ts create mode 100644 packages/cspell-io/src/CSpellIONode.ts create mode 100644 packages/cspell-io/src/CSpellIOWeb.ts create mode 100644 packages/cspell-io/src/common/stat.test.ts create mode 100644 packages/cspell-io/src/common/stat.ts create mode 100644 packages/cspell-io/src/errors/ErrorNotImplemented.ts create mode 100644 packages/cspell-io/src/handlers/node/file.ts create mode 100644 packages/cspell-io/src/models/Stats.ts create mode 100644 packages/cspell-io/src/models/index.ts rename packages/cspell-io/src/{ => node}/file/FetchError.test.ts (100%) rename packages/cspell-io/src/{ => node}/file/FetchError.ts (100%) rename packages/cspell-io/src/{ => node}/file/fetch.test.ts (100%) rename packages/cspell-io/src/{ => node}/file/fetch.ts (100%) rename packages/cspell-io/src/{ => node}/file/fileReader.test.ts (98%) rename packages/cspell-io/src/{ => node}/file/fileReader.ts (100%) rename packages/cspell-io/src/{ => node}/file/fileWriter.test.ts (84%) rename packages/cspell-io/src/{ => node}/file/fileWriter.ts (100%) create mode 100644 packages/cspell-io/src/node/file/index.ts rename packages/cspell-io/src/{ => node}/file/stat.test.ts (100%) rename packages/cspell-io/src/{ => node}/file/stat.ts (64%) rename packages/cspell-io/src/{ => node}/file/util.test.ts (100%) rename packages/cspell-io/src/{ => node}/file/util.ts (77%) create mode 100644 packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts create mode 100644 packages/cspell-io/src/requests/RequestFsReadFile.ts create mode 100644 packages/cspell-io/src/requests/RequestFsReadFileSync.ts create mode 100644 packages/cspell-io/src/requests/RequestZlibInflate.ts create mode 100644 packages/cspell-io/src/requests/index.ts create mode 100644 packages/cspell-service-bus/src/requestFactory.ts diff --git a/cspell.code-workspace b/cspell.code-workspace index 01e3a1e0496..15326b8380a 100644 --- a/cspell.code-workspace +++ b/cspell.code-workspace @@ -31,6 +31,9 @@ { "path": "packages/cspell-grammar" }, + { + "path": "packages/cspell-io" + }, { "path": "packages/cspell-json-reporter" }, diff --git a/packages/cspell-io/package-lock.json b/packages/cspell-io/package-lock.json index 6aca0437c8c..4a19fb73875 100644 --- a/packages/cspell-io/package-lock.json +++ b/packages/cspell-io/package-lock.json @@ -9,6 +9,7 @@ "version": "6.4.2", "license": "MIT", "dependencies": { + "@cspell/cspell-service-bus": "^6.4.2", "@types/node-fetch": "^2.6.2", "node-fetch": "^2.6.7" }, @@ -577,6 +578,14 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@cspell/cspell-service-bus": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/@cspell/cspell-service-bus/-/cspell-service-bus-6.4.2.tgz", + "integrity": "sha512-/XfXQ/yWHeXWVGPPxztBncmDNYoU6tPJ6nS55D0PLgarCqbyJQuI9ksLWJagG2EKPTBFHpzilkYiwNrAGntLVw==", + "engines": { + "node": ">=14" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -4019,6 +4028,11 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "@cspell/cspell-service-bus": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/@cspell/cspell-service-bus/-/cspell-service-bus-6.4.2.tgz", + "integrity": "sha512-/XfXQ/yWHeXWVGPPxztBncmDNYoU6tPJ6nS55D0PLgarCqbyJQuI9ksLWJagG2EKPTBFHpzilkYiwNrAGntLVw==" + }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", diff --git a/packages/cspell-io/package.json b/packages/cspell-io/package.json index 1c844ad94f2..e875978f8cc 100644 --- a/packages/cspell-io/package.json +++ b/packages/cspell-io/package.json @@ -46,6 +46,7 @@ "rimraf": "^3.0.2" }, "dependencies": { + "@cspell/cspell-service-bus": "^6.4.2", "@types/node-fetch": "^2.6.2", "node-fetch": "^2.6.7" } diff --git a/packages/cspell-io/src/CSpellIO.ts b/packages/cspell-io/src/CSpellIO.ts new file mode 100644 index 00000000000..bc8a09ea749 --- /dev/null +++ b/packages/cspell-io/src/CSpellIO.ts @@ -0,0 +1,10 @@ +import type { Stats } from './models'; + +export interface CSpellIO { + readFile(uriOrFilename: string): Promise; + readFileSync(uriOrFilename: string): string; + writeFile(uriOrFilename: string, content: string): Promise; + getStat(uriOrFilename: string): Promise; + getStatSync(uriOrFilename: string): Stats; + compareStats(left: Stats, right: Stats): number; +} diff --git a/packages/cspell-io/src/CSpellIONode.test.ts b/packages/cspell-io/src/CSpellIONode.test.ts new file mode 100644 index 00000000000..e5cacad2380 --- /dev/null +++ b/packages/cspell-io/src/CSpellIONode.test.ts @@ -0,0 +1,37 @@ +import { CSpellIONode } from './CSpellIONode'; + +// const sc = expect.stringContaining; + +describe('CSpellIONode', () => { + test('constructor', () => { + const cspellIo = new CSpellIONode(); + expect(cspellIo).toBeDefined(); + }); + + test.each` + filename | expected + ${__filename} | ${'Method readFile is not supported.'} + `('readFile', async ({ filename, expected }) => { + const cspellIo = new CSpellIONode(); + await expect(() => cspellIo.readFile(filename)).toThrow(expected); + }); + + // readFile(_uriOrFilename: string): Promise { + // throw new ErrorNotImplemented('readFile'); + // } + // readFileSync(_uriOrFilename: string): string { + // throw new ErrorNotImplemented('readFileSync'); + // } + // writeFile(_uriOrFilename: string, _content: string): Promise { + // throw new ErrorNotImplemented('writeFile'); + // } + // getStat(_uriOrFilename: string): Promise { + // throw new ErrorNotImplemented('getStat'); + // } + // getStatSync(_uriOrFilename: string): Stats { + // throw new ErrorNotImplemented('getStatSync'); + // } + // compareStats(left: Stats, right: Stats): number { + // return compareStats(left, right); + // } +}); diff --git a/packages/cspell-io/src/CSpellIONode.ts b/packages/cspell-io/src/CSpellIONode.ts new file mode 100644 index 00000000000..f51c0f719db --- /dev/null +++ b/packages/cspell-io/src/CSpellIONode.ts @@ -0,0 +1,25 @@ +import { CSpellIO } from './CSpellIO'; +import { ErrorNotImplemented } from './errors/ErrorNotImplemented'; +import { compareStats } from './common/stat'; +import { Stats } from './models/Stats'; + +export class CSpellIONode implements CSpellIO { + readFile(_uriOrFilename: string): Promise { + throw new ErrorNotImplemented('readFile'); + } + readFileSync(_uriOrFilename: string): string { + throw new ErrorNotImplemented('readFileSync'); + } + writeFile(_uriOrFilename: string, _content: string): Promise { + throw new ErrorNotImplemented('writeFile'); + } + getStat(_uriOrFilename: string): Promise { + throw new ErrorNotImplemented('getStat'); + } + getStatSync(_uriOrFilename: string): Stats { + throw new ErrorNotImplemented('getStatSync'); + } + compareStats(left: Stats, right: Stats): number { + return compareStats(left, right); + } +} diff --git a/packages/cspell-io/src/CSpellIOWeb.ts b/packages/cspell-io/src/CSpellIOWeb.ts new file mode 100644 index 00000000000..c9bff68e911 --- /dev/null +++ b/packages/cspell-io/src/CSpellIOWeb.ts @@ -0,0 +1,25 @@ +import { CSpellIO } from './CSpellIO'; +import { ErrorNotImplemented } from './errors/ErrorNotImplemented'; +import { compareStats } from './common/stat'; +import { Stats } from './models/Stats'; + +export class CSpellIOWeb implements CSpellIO { + readFile(_uriOrFilename: string): Promise { + throw new ErrorNotImplemented('readFile'); + } + readFileSync(_uriOrFilename: string): string { + throw new ErrorNotImplemented('readFileSync'); + } + writeFile(_uriOrFilename: string, _content: string): Promise { + throw new ErrorNotImplemented('writeFile'); + } + getStat(_uriOrFilename: string): Promise { + throw new ErrorNotImplemented('getStat'); + } + getStatSync(_uriOrFilename: string): Stats { + throw new ErrorNotImplemented('getStatSync'); + } + compareStats(left: Stats, right: Stats): number { + return compareStats(left, right); + } +} diff --git a/packages/cspell-io/src/common/stat.test.ts b/packages/cspell-io/src/common/stat.test.ts new file mode 100644 index 00000000000..b6b25369b2a --- /dev/null +++ b/packages/cspell-io/src/common/stat.test.ts @@ -0,0 +1,27 @@ +import { compareStats } from './stat'; + +describe('stat', () => { + const stats = { + urlA: { eTag: 'W/"10c5e3c7c73159515d4334813d6ba0255230270d92ebfdbd37151db7a0db5918"', mtimeMs: 0, size: -1 }, + urlB: { eTag: 'W/"10c5e3c7c73159515d4334813d6ba0255230270d92ebfdbd37151db7a0dbffff"', mtimeMs: 0, size: -1 }, + file1: { mtimeMs: 1658757408444.0342, size: 1886 }, + file2: { mtimeMs: 1658757408444.0342, size: 2886 }, + file3: { mtimeMs: 1758757408444.0342, size: 1886 }, + }; + + test.each` + left | right | expected + ${stats.urlA} | ${stats.urlA} | ${0} + ${stats.urlA} | ${stats.urlB} | ${-1} + ${stats.urlA} | ${stats.file1} | ${1} + ${stats.file1} | ${stats.file1} | ${0} + ${stats.file1} | ${stats.file2} | ${-1} + ${stats.file1} | ${stats.file3} | ${-1} + ${stats.file2} | ${stats.file1} | ${1} + ${stats.file3} | ${stats.file1} | ${1} + ${stats.file2} | ${stats.file3} | ${1} + `('getStat $left <> $right', async ({ left, right, expected }) => { + const r = compareStats(left, right); + expect(r).toEqual(expected); + }); +}); diff --git a/packages/cspell-io/src/common/stat.ts b/packages/cspell-io/src/common/stat.ts new file mode 100644 index 00000000000..f1415eba463 --- /dev/null +++ b/packages/cspell-io/src/common/stat.ts @@ -0,0 +1,14 @@ +import { Stats } from '../models/Stats'; + +/** + * Compare two Stats to see if they have the same value. + * @param left - Stats + * @param right - Stats + * @returns 0 - equal; 1 - left > right; -1 left < right + */ +export function compareStats(left: Stats, right: Stats): number { + if (left === right) return 0; + if (left.eTag || right.eTag) return left.eTag === right.eTag ? 0 : (left.eTag || '') < (right.eTag || '') ? -1 : 1; + const diff = left.size - right.size || left.mtimeMs - right.mtimeMs; + return diff < 0 ? -1 : diff > 0 ? 1 : 0; +} diff --git a/packages/cspell-io/src/errors/ErrorNotImplemented.ts b/packages/cspell-io/src/errors/ErrorNotImplemented.ts new file mode 100644 index 00000000000..b33c83721db --- /dev/null +++ b/packages/cspell-io/src/errors/ErrorNotImplemented.ts @@ -0,0 +1,5 @@ +export class ErrorNotImplemented extends Error { + constructor(readonly method: string) { + super(`Method ${method} is not supported.`); + } +} diff --git a/packages/cspell-io/src/file/index.ts b/packages/cspell-io/src/file/index.ts index 14831aa095f..bc4b376c500 100644 --- a/packages/cspell-io/src/file/index.ts +++ b/packages/cspell-io/src/file/index.ts @@ -1,4 +1,3 @@ -export { readFile, readFileSync } from './fileReader'; -export { writeToFile, writeToFileIterable, writeToFileIterableP } from './fileWriter'; -export { getStat, getStatSync } from './stat'; -export type { Stats } from './stat'; +export { readFile, readFileSync } from './../node/file'; +export { writeToFile, writeToFileIterable, writeToFileIterableP } from './../node/file'; +export { getStat, getStatSync } from './../node/file'; diff --git a/packages/cspell-io/src/handlers/node/file.ts b/packages/cspell-io/src/handlers/node/file.ts new file mode 100644 index 00000000000..a095146acdd --- /dev/null +++ b/packages/cspell-io/src/handlers/node/file.ts @@ -0,0 +1,58 @@ +import { createRequestHandler, ServiceBus, createResponse, isServiceResponseSuccess } from '@cspell/cspell-service-bus'; +import { RequestFsReadBinaryFile, RequestFsReadFile, RequestZlibInflate } from '../../requests'; +import { promises as fs } from 'fs'; +import { deflateSync } from 'zlib'; +import { isZipped } from '../../node/file/util'; + +/** + * Handle Binary File Reads + */ +const handleRequestFsReadBinaryFile = createRequestHandler( + RequestFsReadBinaryFile, + ({ params }) => createResponse(fs.readFile(params.filename)), + undefined, + 'Node: Read Binary File.' +); + +const handleRequestFsReadFile = createRequestHandler( + RequestFsReadFile, + ({ params }) => createResponse(fs.readFile(params.filename, 'utf-8')), + RequestFsReadFile.type, + 'Node: Read Text File.' +); + +const handleRequestZlibInflate = createRequestHandler( + RequestZlibInflate, + ({ params }) => createResponse(deflateSync(params.data).toString('utf-8')), + RequestZlibInflate.type, + 'Node: gz deflate.' +); + +const handleRequestFsReadFileGz = createRequestHandler( + RequestFsReadFile, + (req, next, dispatcher) => { + const { filename } = req.params; + if (!isZipped(filename)) return next(req); + const result = dispatcher.dispatch(RequestFsReadBinaryFile.create({ filename })); + return isServiceResponseSuccess(result) + ? createResponse(result.value.then((buf) => deflateSync(buf).toString('utf-8'))) + : result; + }, + undefined, + 'Node: Read GZ Text File.' +); + +export function registerHandlers(serviceBus: ServiceBus) { + /** + * Handlers are in order of low to high level + * Order is VERY important. + */ + const handlers = [ + handleRequestFsReadBinaryFile, + handleRequestFsReadFile, + handleRequestZlibInflate, + handleRequestFsReadFileGz, + ]; + + handlers.forEach((handler) => serviceBus.addHandler(handler)); +} diff --git a/packages/cspell-io/src/index.ts b/packages/cspell-io/src/index.ts index bd6fac42fa0..e2450effd9e 100644 --- a/packages/cspell-io/src/index.ts +++ b/packages/cspell-io/src/index.ts @@ -1,2 +1,13 @@ -export * from './file'; export { toArray as asyncIterableToArray } from './async/asyncIterable'; +export { + getStat, + getStatSync, + readFile, + readFileSync, + writeToFile, + writeToFileIterable, + writeToFileIterableP, +} from './file'; +export type { Stats } from './models/Stats'; +export type { CSpellIO } from './CSpellIO'; +export { CSpellIONode } from './CSpellIONode'; diff --git a/packages/cspell-io/src/models/Stats.ts b/packages/cspell-io/src/models/Stats.ts new file mode 100644 index 00000000000..074a5d4d6d2 --- /dev/null +++ b/packages/cspell-io/src/models/Stats.ts @@ -0,0 +1,18 @@ +/** + * Subset of definition from the Node definition to avoid a dependency upon a specific version of Node + */ + +export interface Stats { + /** + * Size of file in byes, -1 if unknown. + */ + size: number; + /** + * Modification time, 0 if unknown. + */ + mtimeMs: number; + /** + * Used by web requests to see if a resource has changed. + */ + eTag?: string | undefined; +} diff --git a/packages/cspell-io/src/models/index.ts b/packages/cspell-io/src/models/index.ts new file mode 100644 index 00000000000..06b17a036fd --- /dev/null +++ b/packages/cspell-io/src/models/index.ts @@ -0,0 +1 @@ +export type { Stats } from './Stats'; diff --git a/packages/cspell-io/src/file/FetchError.test.ts b/packages/cspell-io/src/node/file/FetchError.test.ts similarity index 100% rename from packages/cspell-io/src/file/FetchError.test.ts rename to packages/cspell-io/src/node/file/FetchError.test.ts diff --git a/packages/cspell-io/src/file/FetchError.ts b/packages/cspell-io/src/node/file/FetchError.ts similarity index 100% rename from packages/cspell-io/src/file/FetchError.ts rename to packages/cspell-io/src/node/file/FetchError.ts diff --git a/packages/cspell-io/src/file/fetch.test.ts b/packages/cspell-io/src/node/file/fetch.test.ts similarity index 100% rename from packages/cspell-io/src/file/fetch.test.ts rename to packages/cspell-io/src/node/file/fetch.test.ts diff --git a/packages/cspell-io/src/file/fetch.ts b/packages/cspell-io/src/node/file/fetch.ts similarity index 100% rename from packages/cspell-io/src/file/fetch.ts rename to packages/cspell-io/src/node/file/fetch.ts diff --git a/packages/cspell-io/src/file/fileReader.test.ts b/packages/cspell-io/src/node/file/fileReader.test.ts similarity index 98% rename from packages/cspell-io/src/file/fileReader.test.ts rename to packages/cspell-io/src/node/file/fileReader.test.ts index 3117c2a7a70..687582e8c64 100644 --- a/packages/cspell-io/src/file/fileReader.test.ts +++ b/packages/cspell-io/src/node/file/fileReader.test.ts @@ -3,7 +3,7 @@ import { promises as fs } from 'fs'; import * as path from 'path'; import { isUrlLike, toURL } from './util'; -const root = path.join(__dirname, '../..'); +const root = path.join(__dirname, '../../..'); const oc = expect.objectContaining; describe('Validate the fileReader', () => { diff --git a/packages/cspell-io/src/file/fileReader.ts b/packages/cspell-io/src/node/file/fileReader.ts similarity index 100% rename from packages/cspell-io/src/file/fileReader.ts rename to packages/cspell-io/src/node/file/fileReader.ts diff --git a/packages/cspell-io/src/file/fileWriter.test.ts b/packages/cspell-io/src/node/file/fileWriter.test.ts similarity index 84% rename from packages/cspell-io/src/file/fileWriter.test.ts rename to packages/cspell-io/src/node/file/fileWriter.test.ts index feea71d46eb..307f91e9933 100644 --- a/packages/cspell-io/src/file/fileWriter.test.ts +++ b/packages/cspell-io/src/node/file/fileWriter.test.ts @@ -4,12 +4,15 @@ import * as path from 'path'; import { mkdirp } from 'fs-extra'; import { readFile } from './fileReader'; +const root = path.join(__dirname, '..', '..'); +const tempDir = path.join(root, 'temp'); + describe('Validate the writer', () => { test('tests writing data and reading it back.', async () => { // cspell:ignore éåáí const text = loremIpsum({ count: 1000, format: 'plain', units: 'words' }) + ' éåáí'; const data = text.split(/\b/); - const filename = path.join(__dirname, '..', '..', 'temp', 'tests-writing-an-observable.txt'); + const filename = path.join(tempDir, 'tests-writing-an-observable.txt'); await mkdirp(path.dirname(filename)); await fileWriter.writeToFileIterableP(filename, data); @@ -20,7 +23,7 @@ describe('Validate the writer', () => { test('tests writing data and reading it back. gz', async () => { const text = loremIpsum({ count: 1000, format: 'plain', units: 'words' }) + ' éåáí'; const data = text.split(/\b/); - const filename = path.join(__dirname, '..', '..', 'temp', 'tests-writing-an-observable.txt.gz'); + const filename = path.join(tempDir, 'tests-writing-an-observable.txt.gz'); await mkdirp(path.dirname(filename)); await fileWriter.writeToFileIterableP(filename, data); @@ -30,7 +33,7 @@ describe('Validate the writer', () => { test('tests writeToFile', async () => { const text = loremIpsum({ count: 1000, format: 'plain', units: 'words' }) + ' éåáí'; - const filename = path.join(__dirname, '..', '..', 'temp', 'tests-writing.txt'); + const filename = path.join(tempDir, 'tests-writing.txt'); await mkdirp(path.dirname(filename)); const wStream = fileWriter.writeToFile(filename, text); @@ -45,7 +48,7 @@ describe('Validate the writer', () => { test('tests writeToFile zip', async () => { const text = loremIpsum({ count: 1000, format: 'plain', units: 'words' }) + ' éåáí'; - const filename = path.join(__dirname, '..', '..', 'temp', 'tests-writing.txt.gz'); + const filename = path.join(tempDir, 'tests-writing.txt.gz'); await mkdirp(path.dirname(filename)); const wStream = fileWriter.writeToFile(filename, text); diff --git a/packages/cspell-io/src/file/fileWriter.ts b/packages/cspell-io/src/node/file/fileWriter.ts similarity index 100% rename from packages/cspell-io/src/file/fileWriter.ts rename to packages/cspell-io/src/node/file/fileWriter.ts diff --git a/packages/cspell-io/src/node/file/index.ts b/packages/cspell-io/src/node/file/index.ts new file mode 100644 index 00000000000..971506fda83 --- /dev/null +++ b/packages/cspell-io/src/node/file/index.ts @@ -0,0 +1,3 @@ +export { readFile, readFileSync } from './fileReader'; +export { writeToFile, writeToFileIterable, writeToFileIterableP } from './fileWriter'; +export { getStat, getStatSync } from './stat'; diff --git a/packages/cspell-io/src/file/stat.test.ts b/packages/cspell-io/src/node/file/stat.test.ts similarity index 100% rename from packages/cspell-io/src/file/stat.test.ts rename to packages/cspell-io/src/node/file/stat.test.ts diff --git a/packages/cspell-io/src/file/stat.ts b/packages/cspell-io/src/node/file/stat.ts similarity index 64% rename from packages/cspell-io/src/file/stat.ts rename to packages/cspell-io/src/node/file/stat.ts index 1ac987c006a..116c2754b1a 100644 --- a/packages/cspell-io/src/file/stat.ts +++ b/packages/cspell-io/src/node/file/stat.ts @@ -1,43 +1,21 @@ import { promises as fs, statSync } from 'fs'; import { format } from 'util'; import { fetchHead } from './fetch'; +import { Stats } from '../../models/Stats'; import { isFileURL, isUrlLike, toURL } from './util'; -/** - * Copied from the Node definition to avoid a dependency upon a specific version of Node - */ -interface StatsBase { - // dev: T; - // ino: T; - // mode: T; - // nlink: T; - // uid: T; - // gid: T; - // rdev: T; - size: T; - // blksize: T; - // blocks: T; - // atimeMs: T; - mtimeMs: T; - // ctimeMs: T; - // birthtimeMs: T; - // atime: Date; - // mtime: Date; - // ctime: Date; - // birthtime: Date; - eTag?: string | undefined; -} - export async function getStat(filenameOrUri: string): Promise { if (isUrlLike(filenameOrUri)) { const url = toURL(filenameOrUri); if (!isFileURL(url)) { try { const headers = await fetchHead(url); + const eTag = headers.get('etag') || undefined; + const guessSize = Number.parseInt(headers.get('content-length') || '0', 10); return { - size: Number.parseInt(headers.get('content-length') || '0', 10), + size: eTag ? -1 : guessSize, mtimeMs: 0, - eTag: headers.get('etag') || undefined, + eTag, }; } catch (e) { return toError(e); @@ -65,5 +43,3 @@ function isErrnoException(e: unknown | NodeJS.ErrnoException): e is NodeJS.Errno const err = e as NodeJS.ErrnoException; return err.message !== undefined && err.name !== undefined; } - -export type Stats = StatsBase; diff --git a/packages/cspell-io/src/file/util.test.ts b/packages/cspell-io/src/node/file/util.test.ts similarity index 100% rename from packages/cspell-io/src/file/util.test.ts rename to packages/cspell-io/src/node/file/util.test.ts diff --git a/packages/cspell-io/src/file/util.ts b/packages/cspell-io/src/node/file/util.ts similarity index 77% rename from packages/cspell-io/src/file/util.ts rename to packages/cspell-io/src/node/file/util.ts index 5e10b31e300..0b2a25a47cd 100644 --- a/packages/cspell-io/src/file/util.ts +++ b/packages/cspell-io/src/node/file/util.ts @@ -1,4 +1,4 @@ -import { pathToFileURL, URL } from 'url'; +import { pathToFileURL } from 'url'; const isZippedRegExp = /\.gz($|[?#])/i; @@ -19,5 +19,9 @@ export function isFileURL(url: URL): boolean { return url.protocol === 'file:'; } export function toURL(filename: string | URL): URL { - return filename instanceof URL ? filename : isUrlLike(filename) ? new URL(filename) : pathToFileURL(filename); + return filename instanceof URL || typeof filename !== 'string' + ? filename + : isUrlLike(filename) + ? new URL(filename) + : pathToFileURL(filename); } diff --git a/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts b/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts new file mode 100644 index 00000000000..eb254bac01f --- /dev/null +++ b/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts @@ -0,0 +1,7 @@ +import { requestFactory } from '@cspell/cspell-service-bus'; + +const RequestType = 'fs:readBinaryFile' as const; +interface RequestParams { + readonly filename: string; +} +export const RequestFsReadBinaryFile = requestFactory>(RequestType); diff --git a/packages/cspell-io/src/requests/RequestFsReadFile.ts b/packages/cspell-io/src/requests/RequestFsReadFile.ts new file mode 100644 index 00000000000..1f5de93a8d7 --- /dev/null +++ b/packages/cspell-io/src/requests/RequestFsReadFile.ts @@ -0,0 +1,7 @@ +import { requestFactory } from '@cspell/cspell-service-bus'; + +const RequestType = 'fs:readFile' as const; +interface RequestParams { + readonly filename: string; +} +export const RequestFsReadFile = requestFactory>(RequestType); diff --git a/packages/cspell-io/src/requests/RequestFsReadFileSync.ts b/packages/cspell-io/src/requests/RequestFsReadFileSync.ts new file mode 100644 index 00000000000..bd956e96590 --- /dev/null +++ b/packages/cspell-io/src/requests/RequestFsReadFileSync.ts @@ -0,0 +1,7 @@ +import { requestFactory } from '@cspell/cspell-service-bus'; + +const RequestType = 'fs:readFileSync' as const; +interface RequestParams { + readonly filename: string; +} +export const RequestFsReadFile = requestFactory(RequestType); diff --git a/packages/cspell-io/src/requests/RequestZlibInflate.ts b/packages/cspell-io/src/requests/RequestZlibInflate.ts new file mode 100644 index 00000000000..532db6a9e93 --- /dev/null +++ b/packages/cspell-io/src/requests/RequestZlibInflate.ts @@ -0,0 +1,7 @@ +import { requestFactory } from '@cspell/cspell-service-bus'; + +const RequestType = 'zlib:inflate' as const; +interface RequestParams { + readonly data: Buffer; +} +export const RequestZlibInflate = requestFactory(RequestType); diff --git a/packages/cspell-io/src/requests/index.ts b/packages/cspell-io/src/requests/index.ts new file mode 100644 index 00000000000..1de5dea1ee7 --- /dev/null +++ b/packages/cspell-io/src/requests/index.ts @@ -0,0 +1,3 @@ +export { RequestFsReadFile } from './RequestFsReadFile'; +export { RequestZlibInflate } from './RequestZlibInflate'; +export { RequestFsReadBinaryFile } from './RequestFsReadBinaryFile'; diff --git a/packages/cspell-service-bus/src/SystemServiceBus.test.ts b/packages/cspell-service-bus/src/SystemServiceBus.test.ts index 35f833020d7..109e2558bde 100644 --- a/packages/cspell-service-bus/src/SystemServiceBus.test.ts +++ b/packages/cspell-service-bus/src/SystemServiceBus.test.ts @@ -4,9 +4,10 @@ import { createResponseFail, isServiceResponseFailure, isServiceResponseSuccess, - ServiceRequest, + RequestResponseType, ServiceRequestFactory, } from './request'; +import { requestFactory } from './requestFactory'; import { createSystemServiceBus, RequestCreateSubsystemFactory, @@ -14,39 +15,21 @@ import { } from './SystemServiceBus'; const TypeRequestFsReadFile = 'fs:readFile' as const; -class RequestFsReadFile extends ServiceRequest { - static type = TypeRequestFsReadFile; - private constructor(readonly uri: string) { - super(TypeRequestFsReadFile); - } - static is(req: ServiceRequest): req is RequestFsReadFile { - return req instanceof RequestFsReadFile; - } - static create(uri: string) { - return new RequestFsReadFile(uri); - } -} +const RequestFsReadFile = requestFactory( + TypeRequestFsReadFile +); const TypeRequestZlibInflate = 'zlib:inflate' as const; -class RequestZlibInflate extends ServiceRequest { - static type = TypeRequestZlibInflate; - private constructor(readonly data: string) { - super(TypeRequestZlibInflate); - } - static is(req: ServiceRequest): req is RequestZlibInflate { - return req instanceof RequestZlibInflate; - } - static create(data: string) { - return new RequestZlibInflate(data); - } -} +const RequestZlibInflate = requestFactory( + TypeRequestZlibInflate +); const knownRequestTypes = { [RequestRegisterHandlerFactory.type]: RequestRegisterHandlerFactory, [RequestCreateSubsystemFactory.type]: RequestCreateSubsystemFactory, [RequestFsReadFile.type]: RequestFsReadFile, [RequestZlibInflate.type]: RequestZlibInflate, -}; +} as const; describe('SystemServiceBus', () => { test('createSystemServiceBus', () => { @@ -69,35 +52,35 @@ describe('SystemServiceBus Behavior', () => { serviceBus.createSubsystem('File System', 'fs:'); serviceBus.createSubsystem('ZLib', 'zlib:'); serviceBus.createSubsystem('Path', 'path:'); - serviceBus.registerRequestHandler(RequestFsReadFile, (req) => createResponse(`read file: ${req.uri}`)); + serviceBus.registerRequestHandler(RequestFsReadFile, (req) => createResponse(`read file: ${req.params.uri}`)); serviceBus.registerRequestHandler(RequestFsReadFile, (req, next) => - /https?:/.test(req.uri) ? createResponse(`fetch http: ${req.uri}`) : next(req) + /https?:/.test(req.params.uri) ? createResponse(`fetch http: ${req.params.uri}`) : next(req) ); serviceBus.registerRequestHandler( RequestFsReadFile, (req, next, dispatcher) => { - if (!req.uri.endsWith('.gz')) { + if (!req.params.uri.endsWith('.gz')) { return next(req); } const fileRes = next(req); - if (!isServiceResponseSuccess(fileRes)) return fileRes; - const decompressRes = dispatcher.dispatch(RequestZlibInflate.create(fileRes.value)); + if (!isServiceResponseSuccess>(fileRes)) return fileRes; + const decompressRes = dispatcher.dispatch(RequestZlibInflate.create({ data: fileRes.value })); if (isServiceResponseFailure(decompressRes)) { - return createResponseFail(RequestFsReadFile, decompressRes.error); + return createResponseFail(req, decompressRes.error); } assert(decompressRes.value); return createResponse(decompressRes.value); }, RequestFsReadFile.type + '/zip' ); - serviceBus.registerRequestHandler(RequestZlibInflate, (req) => createResponse(`Inflate: ${req.data}`)); + serviceBus.registerRequestHandler(RequestZlibInflate, (req) => createResponse(`Inflate: ${req.params.data}`)); test.each` - request | expected - ${RequestFsReadFile.create('file://my_file.txt')} | ${{ value: 'read file: file://my_file.txt' }} - ${RequestFsReadFile.create('https://www.example.com/my_file.txt')} | ${{ value: 'fetch http: https://www.example.com/my_file.txt' }} - ${RequestFsReadFile.create('https://www.example.com/my_dict.trie.gz')} | ${{ value: 'Inflate: fetch http: https://www.example.com/my_dict.trie.gz' }} - ${{ type: 'zlib:compress' }} | ${{ error: Error('Unhandled Request: zlib:compress') }} + request | expected + ${RequestFsReadFile.create({ uri: 'file://my_file.txt' })} | ${{ value: 'read file: file://my_file.txt' }} + ${RequestFsReadFile.create({ uri: 'https://www.example.com/my_file.txt' })} | ${{ value: 'fetch http: https://www.example.com/my_file.txt' }} + ${RequestFsReadFile.create({ uri: 'https://www.example.com/my_dict.trie.gz' })} | ${{ value: 'Inflate: fetch http: https://www.example.com/my_dict.trie.gz' }} + ${{ type: 'zlib:compress' }} | ${{ error: Error('Unhandled Request: zlib:compress') }} `('dispatch requests', ({ request, expected }) => { expect(serviceBus.dispatch(request)).toEqual(expected); }); diff --git a/packages/cspell-service-bus/src/SystemServiceBus.ts b/packages/cspell-service-bus/src/SystemServiceBus.ts index 0dc249e73ee..6db42202133 100644 --- a/packages/cspell-service-bus/src/SystemServiceBus.ts +++ b/packages/cspell-service-bus/src/SystemServiceBus.ts @@ -4,12 +4,19 @@ import { createServiceBus, Dispatcher, Handler, + HandleRequest, HandleRequestFn, - HandleRequestKnown, HandlerNext, ServiceBus, } from './bus'; -import { createResponse, RequestResponseType, ServiceRequest, ServiceRequestFactory } from './request'; +import { + createResponse, + RequestResponseType, + ServiceRequest, + ServiceRequestFactory, + ServiceRequestFactoryRequestType, +} from './request'; +import { requestFactory } from './requestFactory'; export interface SystemServiceBus extends Dispatcher { registerHandler(requestPrefix: string, handler: Handler): void; @@ -36,7 +43,7 @@ class SystemServiceBusImpl implements SystemServiceBus { private bindDefaultHandlers() { this.serviceBus.addHandler( createRequestHandler(RequestCreateSubsystemFactory, (req) => { - const { name, requestPattern } = req; + const { name, requestPattern } = req.params; const sub = createSubsystemServiceBus(name, requestPattern); this._subsystems.push(sub); this.serviceBus.addHandler(sub.handler); @@ -50,13 +57,13 @@ class SystemServiceBusImpl implements SystemServiceBus { } createSubsystem(name: string, requestPattern: string | RegExp): SubsystemServiceBus { - const res = this.dispatch(RequestCreateSubsystemFactory.create(name, requestPattern)); + const res = this.dispatch(RequestCreateSubsystemFactory.create({ name, requestPattern })); assert(res?.value); return res.value; } registerHandler(requestPrefix: string, handler: Handler): void { - const request = RequestRegisterHandlerFactory.create(requestPrefix, handler); + const request = RequestRegisterHandlerFactory.create({ requestPrefix, handler }); this.serviceBus.dispatch(request); } @@ -79,38 +86,18 @@ export function createSystemServiceBus(): SystemServiceBus { } const TypeRequestRegisterHandler = 'System:RegisterHandler' as const; -export class RequestRegisterHandlerFactory extends ServiceRequest< +export const RequestRegisterHandlerFactory = requestFactory< typeof TypeRequestRegisterHandler, + { readonly requestPrefix: string; readonly handler: Handler }, SubsystemServiceBus -> { - static type = TypeRequestRegisterHandler; - private constructor(readonly requestPrefix: string, readonly handler: Handler) { - super(RequestRegisterHandlerFactory.type); - } - static is(req: ServiceRequest): req is RequestRegisterHandlerFactory { - return req instanceof RequestRegisterHandlerFactory; - } - static create(requestPrefix: string, handler: Handler) { - return new RequestRegisterHandlerFactory(requestPrefix, handler); - } -} +>(TypeRequestRegisterHandler); const TypeRequestCreateSubsystem = 'System:CreateSubsystem' as const; -export class RequestCreateSubsystemFactory extends ServiceRequest< +export const RequestCreateSubsystemFactory = requestFactory< typeof TypeRequestCreateSubsystem, + { readonly name: string; readonly requestPattern: string | RegExp }, SubsystemServiceBus -> { - static type = TypeRequestCreateSubsystem; - private constructor(readonly name: string, readonly requestPattern: string | RegExp) { - super(RequestCreateSubsystemFactory.type); - } - static is(req: ServiceRequest): req is RequestCreateSubsystemFactory { - return req instanceof RequestCreateSubsystemFactory; - } - static create(name: string, requestPattern: string | RegExp) { - return new RequestCreateSubsystemFactory(name, requestPattern); - } -} +>(TypeRequestCreateSubsystem); interface SubsystemServiceBus extends Dispatcher { readonly name: string; @@ -145,16 +132,16 @@ class SubsystemServiceBusImpl extends ServiceBus implements SubsystemServiceBus } handleRegistrationReq( - request: RequestRegisterHandlerFactory, - next: HandleRequestKnown + request: ServiceRequestFactoryRequestType, + next: HandleRequest ) { // console.log(`${this.name}.handleRegistrationReq %o`, request); - if (!this.canHandleType(request.requestPrefix)) { + if (!this.canHandleType(request.params.requestPrefix)) { // console.log(`${this.name}.handleRegistrationReq skip`); return next(request); } // console.log(`${this.name}.handleRegistrationReq add ***`); - this.addHandler(request.handler); + this.addHandler(request.params.handler); return createResponse(this); } diff --git a/packages/cspell-service-bus/src/__snapshots__/index.test.ts.snap b/packages/cspell-service-bus/src/__snapshots__/index.test.ts.snap index 28a31446b94..538508fb820 100644 --- a/packages/cspell-service-bus/src/__snapshots__/index.test.ts.snap +++ b/packages/cspell-service-bus/src/__snapshots__/index.test.ts.snap @@ -5,5 +5,6 @@ Map { "ServiceRequest" => "function", "ServiceBus" => "function", "createServiceBus" => "function", + "requestFactory" => "function", } `; diff --git a/packages/cspell-service-bus/src/bus.test.ts b/packages/cspell-service-bus/src/bus.test.ts index 3b8d2e20962..a2c95300f5a 100644 --- a/packages/cspell-service-bus/src/bus.test.ts +++ b/packages/cspell-service-bus/src/bus.test.ts @@ -4,7 +4,7 @@ import { createResponse as response, ServiceRequest, ServiceResponse } from './r function calcFib(request: FibRequest): ServiceResponse { let a = 0, b = 1; - let n = request.fib; + let n = request.params.fib; while (--n >= 0) { const c = a + b; @@ -18,46 +18,46 @@ function calcFib(request: FibRequest): ServiceResponse { } const TypeRequestFib = 'Computations:calc-fib' as const; -class FibRequest extends ServiceRequest { +class FibRequest extends ServiceRequest { static type = TypeRequestFib; - private constructor(readonly fib: number) { - super(TypeRequestFib); + private constructor(params: { fib: number }) { + super(TypeRequestFib, params); } static is(req: ServiceRequest): req is FibRequest { return req instanceof FibRequest; } - static create(fib: number) { - return new FibRequest(fib); + static create(params: { fib: number }) { + return new FibRequest(params); } } -class StringLengthRequest extends ServiceRequest<'calc-string-length', number> { +class StringLengthRequest extends ServiceRequest<'calc-string-length', { readonly str: string }, number> { constructor(readonly str: string) { - super('calc-string-length'); + super('calc-string-length', { str }); } static is(req: ServiceRequest): req is StringLengthRequest { return req instanceof StringLengthRequest; } } -class StringToUpperRequest extends ServiceRequest<'toUpper', string> { +class StringToUpperRequest extends ServiceRequest<'toUpper', { readonly str: string }, string> { constructor(readonly str: string) { - super('toUpper'); + super('toUpper', { str }); } static is(req: ServiceRequest): req is StringToUpperRequest { return req instanceof StringToUpperRequest; } } -class DoNotHandleRequest extends ServiceRequest<'Do Not Handle', undefined> { +class DoNotHandleRequest extends ServiceRequest<'Do Not Handle', undefined, undefined> { constructor() { - super('Do Not Handle'); + super('Do Not Handle', undefined); } } -class RetryAgainRequest extends ServiceRequest<'Retry Again Request', undefined> { +class RetryAgainRequest extends ServiceRequest<'Retry Again Request', undefined, undefined> { constructor() { - super('Retry Again Request'); + super('Retry Again Request', undefined); } static is(req: ServiceRequest): req is RetryAgainRequest { return req instanceof RetryAgainRequest; @@ -88,9 +88,9 @@ describe('Service Bus', () => { test.each` request | expected - ${FibRequest.create(6)} | ${response(8)} - ${FibRequest.create(5)} | ${response(5)} - ${FibRequest.create(7)} | ${response(13)} + ${FibRequest.create({ fib: 6 })} | ${response(8)} + ${FibRequest.create({ fib: 5 })} | ${response(5)} + ${FibRequest.create({ fib: 7 })} | ${response(13)} ${new StringLengthRequest('hello')} | ${response(5)} ${new StringToUpperRequest('hello')} | ${response('HELLO')} ${new DoNotHandleRequest()} | ${{ error: Error('Unhandled Request: Do Not Handle') }} diff --git a/packages/cspell-service-bus/src/bus.ts b/packages/cspell-service-bus/src/bus.ts index 3746812fcc4..1fde8209bd7 100644 --- a/packages/cspell-service-bus/src/bus.ts +++ b/packages/cspell-service-bus/src/bus.ts @@ -1,4 +1,11 @@ -import { createResponseFail, IsARequest, RequestResponseType, ServiceRequest, ServiceRequestFactory } from './request'; +import { + createResponseFail, + IsARequest, + RequestResponseType, + ServiceRequest, + ServiceRequestFactory, + ServiceRequestFactoryRequestType, +} from './request'; export interface Dispatcher { dispatch(request: R): RequestResponseType; @@ -69,7 +76,7 @@ export function createServiceBus(handlers: Handler[] = []): ServiceBus { export type HandleRequestFn = ( request: R, - next: HandleRequestKnown, + next: HandleRequest, dispatch: Dispatcher ) => RequestResponseType; @@ -79,10 +86,14 @@ export interface HandleRequest { } export interface HandleRequestKnown { - // eslint-disable-next-line @typescript-eslint/no-explicit-any (request: R): RequestResponseType; } +export type FactoryRequestHandler< + T extends ServiceRequestFactory, + R extends ServiceRequest = ServiceRequestFactoryRequestType +> = HandleRequestKnown; + export interface HandlerNext { (next: HandleRequest): HandleRequest; } diff --git a/packages/cspell-service-bus/src/index.ts b/packages/cspell-service-bus/src/index.ts index 5744d8440b0..33562927b8f 100644 --- a/packages/cspell-service-bus/src/index.ts +++ b/packages/cspell-service-bus/src/index.ts @@ -1,2 +1,9 @@ -export { ServiceRequest } from './request'; -export { ServiceBus, createServiceBus } from './bus'; +export { createRequestHandler, createServiceBus, ServiceBus } from './bus'; +export { + createResponse, + createResponseFail, + isServiceResponseFailure, + isServiceResponseSuccess, + ServiceRequest, +} from './request'; +export { requestFactory } from './requestFactory'; diff --git a/packages/cspell-service-bus/src/request.test.ts b/packages/cspell-service-bus/src/request.test.ts index 9d00549c5ff..6df28d684df 100644 --- a/packages/cspell-service-bus/src/request.test.ts +++ b/packages/cspell-service-bus/src/request.test.ts @@ -31,10 +31,10 @@ describe('request', () => { }); test.each` - request | kind | expected - ${new ServiceRequest('ServiceRequestSync')} | ${BaseServiceRequest} | ${true} - ${new ServiceRequest('ServiceRequestSync')} | ${ServiceRequest} | ${true} - ${{ type: 'static' }} | ${BaseServiceRequest} | ${false} + request | kind | expected + ${new ServiceRequest('ServiceRequestSync', undefined)} | ${BaseServiceRequest} | ${true} + ${new ServiceRequest('ServiceRequestSync', undefined)} | ${ServiceRequest} | ${true} + ${{ type: 'static' }} | ${BaseServiceRequest} | ${false} `('isInstanceOfFn $request.type', ({ request, kind, expected }) => { const fn = isInstanceOfFn(kind); expect(fn(request)).toEqual(expected); diff --git a/packages/cspell-service-bus/src/request.ts b/packages/cspell-service-bus/src/request.ts index 67587f0c3e3..d9483912941 100644 --- a/packages/cspell-service-bus/src/request.ts +++ b/packages/cspell-service-bus/src/request.ts @@ -1,16 +1,17 @@ -export interface ServiceRequest { +export interface ServiceRequest { readonly type: T; + readonly params: P; __r?: ServiceResponseBase; } -class BaseServiceRequest implements ServiceRequest { +class BaseServiceRequest implements ServiceRequest { readonly __r?: ServiceResponseBase; - constructor(readonly type: T) {} + constructor(readonly type: T, readonly params: P) {} } -export class ServiceRequest extends BaseServiceRequest { - constructor(readonly type: T) { - super(type); +export class ServiceRequest extends BaseServiceRequest { + constructor(type: T, params: P) { + super(type, params); } } @@ -60,13 +61,16 @@ export function isInstanceOfFn(constructor: { new (): T }): (t: unknown) => t return (t): t is T => t instanceof constructor; } -export interface ServiceRequestFactory { +export interface ServiceRequestFactory { type: T; is: (r: ServiceRequest | R) => r is R; // eslint-disable-next-line @typescript-eslint/no-explicit-any - create(...params: any[]): R; + create(params: P): R; + __request?: R; } +export type ServiceRequestFactoryRequestType = T extends { __request?: infer R } ? R : never; + export const __testing__ = { BaseServiceRequest, }; diff --git a/packages/cspell-service-bus/src/requestFactory.ts b/packages/cspell-service-bus/src/requestFactory.ts new file mode 100644 index 00000000000..36395fcd392 --- /dev/null +++ b/packages/cspell-service-bus/src/requestFactory.ts @@ -0,0 +1,19 @@ +import { ServiceRequest, ServiceRequestFactory } from './request'; + +export function requestFactory(requestType: T): ServiceRequestFactory> { + type Request = ServiceRequest; + class RequestClass extends ServiceRequest { + static type = requestType; + private constructor(params: P) { + super(requestType, params); + } + static is(req: ServiceRequest): req is RequestClass { + return req instanceof RequestClass && req.type === requestType; + } + static create(params: P) { + return new RequestClass(params); + } + static __request__?: Request; + } + return RequestClass; +} From 5294b2bc6e021dc321a8bc187951f1d2a39418ad Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Tue, 26 Jul 2022 17:56:50 +0200 Subject: [PATCH 2/7] Update index.test.ts.snap --- .../src/__snapshots__/index.test.ts.snap | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/cspell-service-bus/src/__snapshots__/index.test.ts.snap b/packages/cspell-service-bus/src/__snapshots__/index.test.ts.snap index 538508fb820..baa718ea5f0 100644 --- a/packages/cspell-service-bus/src/__snapshots__/index.test.ts.snap +++ b/packages/cspell-service-bus/src/__snapshots__/index.test.ts.snap @@ -2,9 +2,14 @@ exports[`index API 1`] = ` Map { - "ServiceRequest" => "function", - "ServiceBus" => "function", + "createRequestHandler" => "function", "createServiceBus" => "function", + "ServiceBus" => "function", + "createResponse" => "function", + "createResponseFail" => "function", + "isServiceResponseFailure" => "function", + "isServiceResponseSuccess" => "function", + "ServiceRequest" => "function", "requestFactory" => "function", } `; From 6c376e0001bb3fbf56a5fe95aae0c23b7002af37 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Tue, 26 Jul 2022 19:47:56 +0200 Subject: [PATCH 3/7] Add some test helpers --- packages/cspell-io/src/test/helper.ts | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 packages/cspell-io/src/test/helper.ts diff --git a/packages/cspell-io/src/test/helper.ts b/packages/cspell-io/src/test/helper.ts new file mode 100644 index 00000000000..0558b3b0392 --- /dev/null +++ b/packages/cspell-io/src/test/helper.ts @@ -0,0 +1,39 @@ +import * as path from 'path'; +import { mkdirp } from 'fs-extra'; + +const pathPackageRoot = path.join(__dirname, '../..'); +const pathSamples = path.join(pathPackageRoot, 'samples'); +const pathTemp = path.join(pathPackageRoot, 'temp'); + +export function pathToSample(...parts: string[]): string { + return path.resolve(pathSamples, ...parts); +} + +export function pathToRoot(...parts: string[]): string { + return path.resolve(pathPackageRoot, ...parts); +} + +export function makePathToFile(file: string): Promise { + return mkdirp(path.dirname(file)); +} + +export function testNameToDir(testName: string): string { + return `test_${testName.replace(/\s/g, '-').replace(/[^\w.-]/gi, '_')}_test`; +} + +/** + * Calculate a Uri for a path to a temporary directory that will be unique to the current test. + * Note: if a text is not currently running, then it is the path for the test file. + * @param baseFilename - name of file / directory wanted + * @param testFilename - optional full path to a test file. + * @returns full path to the requested temp file. + */ +export function pathToTemp(...parts: string[]): string { + const testState = expect.getState(); + const callerFile = testState.testPath || '.'; + const testFile = path.relative(pathPackageRoot, callerFile); + expect.getState(); + const testName = testState.currentTestName || '.'; + const testDirName = testNameToDir(testName); + return path.resolve(pathTemp, testFile, testDirName, ...parts); +} From f6cedbae80dd4b328e9d59440ef630b216719054 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Tue, 26 Jul 2022 19:48:39 +0200 Subject: [PATCH 4/7] Have tests use the test helper --- .../src/node/file/fileReader.test.ts | 9 ++-- .../src/node/file/fileWriter.test.ts | 53 ++++++------------- 2 files changed, 20 insertions(+), 42 deletions(-) diff --git a/packages/cspell-io/src/node/file/fileReader.test.ts b/packages/cspell-io/src/node/file/fileReader.test.ts index 687582e8c64..242cbb5e964 100644 --- a/packages/cspell-io/src/node/file/fileReader.test.ts +++ b/packages/cspell-io/src/node/file/fileReader.test.ts @@ -1,9 +1,8 @@ import * as fReader from './fileReader'; import { promises as fs } from 'fs'; -import * as path from 'path'; import { isUrlLike, toURL } from './util'; +import { pathToRoot } from '../../test/helper'; -const root = path.join(__dirname, '../../..'); const oc = expect.objectContaining; describe('Validate the fileReader', () => { @@ -23,7 +22,7 @@ describe('Validate the fileReader', () => { ${'samples/cities.txt'} | ${'San Francisco'} ${'samples/cities.txt.gz'} | ${'San Francisco'} `('reading sync files $file', ({ file, contains }) => { - const filename = path.resolve(root, file); + const filename = pathToRoot(file); const content = fReader.readFileSync(filename); expect(content).toContain(contains); }); @@ -33,7 +32,7 @@ describe('Validate the fileReader', () => { ${'samples/cities.txt'} | ${'San Francisco'} ${'samples/cities.txt.gz'} | ${'San Francisco'} `('reading async files $file', async ({ file, contains }) => { - const filename = path.resolve(root, file); + const filename = pathToRoot(file); const content = await fReader.readFile(filename); expect(content).toContain(contains); }); @@ -45,7 +44,7 @@ describe('Validate the fileReader', () => { ${'https://github.com/streetsidesoftware/cspell/raw/main/packages/cspell-io/samples/cities.txt'} | ${'San Francisco'} ${'https://github.com/streetsidesoftware/cspell/raw/main/packages/cspell-io/samples/cities.txt.gz'} | ${'San Francisco'} `('reading URLs files $file', async ({ file, contains }) => { - const filename = isUrlLike(file) ? file : path.resolve(root, file); + const filename = isUrlLike(file) ? file : pathToRoot(file); const url = toURL(filename); const content = await fReader.readFile(url); expect(content).toContain(contains); diff --git a/packages/cspell-io/src/node/file/fileWriter.test.ts b/packages/cspell-io/src/node/file/fileWriter.test.ts index 307f91e9933..40cd9d54754 100644 --- a/packages/cspell-io/src/node/file/fileWriter.test.ts +++ b/packages/cspell-io/src/node/file/fileWriter.test.ts @@ -1,56 +1,35 @@ import * as fileWriter from './fileWriter'; import { loremIpsum } from 'lorem-ipsum'; -import * as path from 'path'; -import { mkdirp } from 'fs-extra'; import { readFile } from './fileReader'; - -const root = path.join(__dirname, '..', '..'); -const tempDir = path.join(root, 'temp'); +import { makePathToFile, pathToTemp } from '../../test/helper'; describe('Validate the writer', () => { - test('tests writing data and reading it back.', async () => { + test.each` + baseFilename + ${'tests-writing-an-observable.txt'} + ${'tests-writing-an-observable.txt.gz'} + `('writeToFileIterableP - writing data and reading it back: $baseFilename', async ({ baseFilename }) => { // cspell:ignore éåáí const text = loremIpsum({ count: 1000, format: 'plain', units: 'words' }) + ' éåáí'; const data = text.split(/\b/); - const filename = path.join(tempDir, 'tests-writing-an-observable.txt'); + const filename = pathToTemp(baseFilename); + await makePathToFile(filename); - await mkdirp(path.dirname(filename)); await fileWriter.writeToFileIterableP(filename, data); const result = await readFile(filename, 'utf8'); expect(result).toBe(text); }); - test('tests writing data and reading it back. gz', async () => { - const text = loremIpsum({ count: 1000, format: 'plain', units: 'words' }) + ' éåáí'; - const data = text.split(/\b/); - const filename = path.join(tempDir, 'tests-writing-an-observable.txt.gz'); - - await mkdirp(path.dirname(filename)); - await fileWriter.writeToFileIterableP(filename, data); - const result = await readFile(filename, 'utf8'); - expect(result).toBe(text); - }); - - test('tests writeToFile', async () => { - const text = loremIpsum({ count: 1000, format: 'plain', units: 'words' }) + ' éåáí'; - const filename = path.join(tempDir, 'tests-writing.txt'); - - await mkdirp(path.dirname(filename)); - const wStream = fileWriter.writeToFile(filename, text); - await new Promise((resolve, reject) => { - wStream.on('close', resolve); - wStream.on('error', reject); - }); - - const result = await readFile(filename, 'utf8'); - expect(result).toBe(text); - }); - - test('tests writeToFile zip', async () => { + test.each` + baseFilename + ${'tests-writing.txt'} + ${'tests-writing.txt.gz'} + `('writeToFile: $baseFilename', async ({ baseFilename }) => { + // cspell:ignore éåáí const text = loremIpsum({ count: 1000, format: 'plain', units: 'words' }) + ' éåáí'; - const filename = path.join(tempDir, 'tests-writing.txt.gz'); + const filename = pathToTemp(baseFilename); + await makePathToFile(filename); - await mkdirp(path.dirname(filename)); const wStream = fileWriter.writeToFile(filename, text); await new Promise((resolve, reject) => { wStream.on('close', resolve); From 4bce54e16c3e2d1154f2cce650dbafd3e7cf17b6 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Tue, 26 Jul 2022 19:49:24 +0200 Subject: [PATCH 5/7] Use URLs in the handlers instead of string. --- packages/cspell-io/src/handlers/node/file.ts | 37 ++++++++++++++++--- .../src/requests/RequestFsReadBinaryFile.ts | 2 +- .../src/requests/RequestFsReadFile.ts | 2 +- .../src/requests/RequestFsReadFileSync.ts | 4 +- .../cspell-io/src/requests/RequestFsStat.ts | 13 +++++++ packages/cspell-io/src/requests/index.ts | 4 +- 6 files changed, 51 insertions(+), 11 deletions(-) create mode 100644 packages/cspell-io/src/requests/RequestFsStat.ts diff --git a/packages/cspell-io/src/handlers/node/file.ts b/packages/cspell-io/src/handlers/node/file.ts index a095146acdd..dcac5ef63ab 100644 --- a/packages/cspell-io/src/handlers/node/file.ts +++ b/packages/cspell-io/src/handlers/node/file.ts @@ -1,26 +1,48 @@ -import { createRequestHandler, ServiceBus, createResponse, isServiceResponseSuccess } from '@cspell/cspell-service-bus'; +import { + createRequestHandler, + ServiceBus, + createResponse, + isServiceResponseSuccess, + createResponseFail, + isServiceResponseFailure, +} from '@cspell/cspell-service-bus'; import { RequestFsReadBinaryFile, RequestFsReadFile, RequestZlibInflate } from '../../requests'; import { promises as fs } from 'fs'; import { deflateSync } from 'zlib'; import { isZipped } from '../../node/file/util'; +import assert from 'assert'; /** * Handle Binary File Reads */ const handleRequestFsReadBinaryFile = createRequestHandler( RequestFsReadBinaryFile, - ({ params }) => createResponse(fs.readFile(params.filename)), + ({ params }) => createResponse(fs.readFile(params.url)), undefined, 'Node: Read Binary File.' ); +/** + * Handle UTF-8 Text File Reads + */ const handleRequestFsReadFile = createRequestHandler( RequestFsReadFile, - ({ params }) => createResponse(fs.readFile(params.filename, 'utf-8')), + (req, _, dispatcher) => { + const { url } = req.params; + const res = dispatcher.dispatch(RequestFsReadBinaryFile.create({ url })); + if (!isServiceResponseSuccess(res)) { + assert(isServiceResponseFailure(res)); + return createResponseFail(req, res.error); + } + return createResponse(res.value.then((buf) => buf.toString('utf-8'))); + }, RequestFsReadFile.type, 'Node: Read Text File.' ); +/** + * Handle deflating gzip data + */ const handleRequestZlibInflate = createRequestHandler( RequestZlibInflate, ({ params }) => createResponse(deflateSync(params.data).toString('utf-8')), @@ -28,12 +50,15 @@ const handleRequestZlibInflate = createRequestHandler( 'Node: gz deflate.' ); +/** + * Handle reading gzip'ed text files. + */ const handleRequestFsReadFileGz = createRequestHandler( RequestFsReadFile, (req, next, dispatcher) => { - const { filename } = req.params; - if (!isZipped(filename)) return next(req); - const result = dispatcher.dispatch(RequestFsReadBinaryFile.create({ filename })); + const { url } = req.params; + if (!isZipped(url)) return next(req); + const result = dispatcher.dispatch(RequestFsReadBinaryFile.create({ url })); return isServiceResponseSuccess(result) ? createResponse(result.value.then((buf) => deflateSync(buf).toString('utf-8'))) : result; diff --git a/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts b/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts index eb254bac01f..2c1f4523316 100644 --- a/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts +++ b/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts @@ -2,6 +2,6 @@ import { requestFactory } from '@cspell/cspell-service-bus'; const RequestType = 'fs:readBinaryFile' as const; interface RequestParams { - readonly filename: string; + readonly url: URL; } export const RequestFsReadBinaryFile = requestFactory>(RequestType); diff --git a/packages/cspell-io/src/requests/RequestFsReadFile.ts b/packages/cspell-io/src/requests/RequestFsReadFile.ts index 1f5de93a8d7..9188a50bc78 100644 --- a/packages/cspell-io/src/requests/RequestFsReadFile.ts +++ b/packages/cspell-io/src/requests/RequestFsReadFile.ts @@ -2,6 +2,6 @@ import { requestFactory } from '@cspell/cspell-service-bus'; const RequestType = 'fs:readFile' as const; interface RequestParams { - readonly filename: string; + readonly url: URL; } export const RequestFsReadFile = requestFactory>(RequestType); diff --git a/packages/cspell-io/src/requests/RequestFsReadFileSync.ts b/packages/cspell-io/src/requests/RequestFsReadFileSync.ts index bd956e96590..894af05a0da 100644 --- a/packages/cspell-io/src/requests/RequestFsReadFileSync.ts +++ b/packages/cspell-io/src/requests/RequestFsReadFileSync.ts @@ -2,6 +2,6 @@ import { requestFactory } from '@cspell/cspell-service-bus'; const RequestType = 'fs:readFileSync' as const; interface RequestParams { - readonly filename: string; + readonly url: URL; } -export const RequestFsReadFile = requestFactory(RequestType); +export const RequestFsReadFileSync = requestFactory(RequestType); diff --git a/packages/cspell-io/src/requests/RequestFsStat.ts b/packages/cspell-io/src/requests/RequestFsStat.ts new file mode 100644 index 00000000000..74df5db239d --- /dev/null +++ b/packages/cspell-io/src/requests/RequestFsStat.ts @@ -0,0 +1,13 @@ +import { requestFactory } from '@cspell/cspell-service-bus'; +import { Stats } from '../models'; + +const RequestTypeStat = 'fs:stat' as const; +interface RequestStatParams { + readonly url: URL; +} +export const RequestFsStat = requestFactory>(RequestTypeStat); + +const RequestTypeStatSync = 'fs:statSync' as const; +export const RequestFsStatSync = requestFactory( + RequestTypeStatSync +); diff --git a/packages/cspell-io/src/requests/index.ts b/packages/cspell-io/src/requests/index.ts index 1de5dea1ee7..acdc4c29b1a 100644 --- a/packages/cspell-io/src/requests/index.ts +++ b/packages/cspell-io/src/requests/index.ts @@ -1,3 +1,5 @@ +export { RequestFsReadBinaryFile } from './RequestFsReadBinaryFile'; export { RequestFsReadFile } from './RequestFsReadFile'; +export { RequestFsReadFileSync } from './RequestFsReadFileSync'; +export { RequestFsStat, RequestFsStatSync } from './RequestFsStat'; export { RequestZlibInflate } from './RequestZlibInflate'; -export { RequestFsReadBinaryFile } from './RequestFsReadBinaryFile'; From fa4bc84414d566bd94fb478f1d2db147562caf18 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Tue, 26 Jul 2022 19:54:27 +0200 Subject: [PATCH 6/7] wire up readFile --- packages/cspell-io/src/CSpellIONode.test.ts | 14 ++++++++--- packages/cspell-io/src/CSpellIONode.ts | 27 ++++++++++++++++++--- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/packages/cspell-io/src/CSpellIONode.test.ts b/packages/cspell-io/src/CSpellIONode.test.ts index e5cacad2380..5c5b80994ab 100644 --- a/packages/cspell-io/src/CSpellIONode.test.ts +++ b/packages/cspell-io/src/CSpellIONode.test.ts @@ -1,6 +1,6 @@ import { CSpellIONode } from './CSpellIONode'; -// const sc = expect.stringContaining; +const sc = expect.stringContaining; describe('CSpellIONode', () => { test('constructor', () => { @@ -10,10 +10,18 @@ describe('CSpellIONode', () => { test.each` filename | expected - ${__filename} | ${'Method readFile is not supported.'} + ${__filename} | ${sc('This bit of text')} `('readFile', async ({ filename, expected }) => { const cspellIo = new CSpellIONode(); - await expect(() => cspellIo.readFile(filename)).toThrow(expected); + await expect(cspellIo.readFile(filename)).resolves.toEqual(expected); + }); + + test.each` + filename | expected + ${__filename} | ${'Unhandled Request: fs:readFileSync' /* sc('This bit of text') */} + `('readFileSync', ({ filename, expected }) => { + const cspellIo = new CSpellIONode(); + expect(() => cspellIo.readFileSync(filename)).toThrow(expected); }); // readFile(_uriOrFilename: string): Promise { diff --git a/packages/cspell-io/src/CSpellIONode.ts b/packages/cspell-io/src/CSpellIONode.ts index f51c0f719db..bed5d2d3f86 100644 --- a/packages/cspell-io/src/CSpellIONode.ts +++ b/packages/cspell-io/src/CSpellIONode.ts @@ -1,13 +1,32 @@ +import { isServiceResponseSuccess, ServiceBus } from '@cspell/cspell-service-bus'; +import { compareStats } from './common/stat'; import { CSpellIO } from './CSpellIO'; import { ErrorNotImplemented } from './errors/ErrorNotImplemented'; -import { compareStats } from './common/stat'; +import { registerHandlers } from './handlers/node/file'; import { Stats } from './models/Stats'; +import { toURL } from './node/file/util'; +import { RequestFsReadFile, RequestFsReadFileSync } from './requests'; export class CSpellIONode implements CSpellIO { - readFile(_uriOrFilename: string): Promise { - throw new ErrorNotImplemented('readFile'); + constructor(readonly serviceBus = new ServiceBus()) { + registerHandlers(serviceBus); + } + + readFile(uriOrFilename: string): Promise { + const url = toURL(uriOrFilename); + const res = this.serviceBus.dispatch(RequestFsReadFile.create({ url })); + if (!isServiceResponseSuccess(res)) { + throw res.error || new ErrorNotImplemented('readFile'); + } + return res.value; } - readFileSync(_uriOrFilename: string): string { + readFileSync(uriOrFilename: string): string { + const url = toURL(uriOrFilename); + const res = this.serviceBus.dispatch(RequestFsReadFileSync.create({ url })); + if (!isServiceResponseSuccess(res)) { + throw res.error || new ErrorNotImplemented('readFileSync'); + } + return res.value; throw new ErrorNotImplemented('readFileSync'); } writeFile(_uriOrFilename: string, _content: string): Promise { From bd04b04c4bf898f5968b99b690d10570bcfaba5b Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Tue, 26 Jul 2022 22:17:25 +0200 Subject: [PATCH 7/7] Support reading http files. --- packages/cspell-io/src/CSpellIONode.test.ts | 62 +++++++++++---- packages/cspell-io/src/CSpellIONode.ts | 44 ++++++++--- packages/cspell-io/src/handlers/node/file.ts | 79 ++++++++++++++----- packages/cspell-io/src/node/file/fetch.ts | 9 +++ .../src/requests/RequestFsReadBinaryFile.ts | 5 ++ .../src/requests/RequestFsWriteFile.ts | 7 ++ packages/cspell-io/src/requests/index.ts | 3 +- 7 files changed, 162 insertions(+), 47 deletions(-) create mode 100644 packages/cspell-io/src/requests/RequestFsWriteFile.ts diff --git a/packages/cspell-io/src/CSpellIONode.test.ts b/packages/cspell-io/src/CSpellIONode.test.ts index 5c5b80994ab..9276f42a187 100644 --- a/packages/cspell-io/src/CSpellIONode.test.ts +++ b/packages/cspell-io/src/CSpellIONode.test.ts @@ -1,6 +1,8 @@ import { CSpellIONode } from './CSpellIONode'; +import { pathToSample as ps } from './test/helper'; const sc = expect.stringContaining; +const oc = expect.objectContaining; describe('CSpellIONode', () => { test('constructor', () => { @@ -9,27 +11,58 @@ describe('CSpellIONode', () => { }); test.each` - filename | expected - ${__filename} | ${sc('This bit of text')} - `('readFile', async ({ filename, expected }) => { + filename | expected + ${__filename} | ${sc('This bit of text')} + ${ps('cities.txt')} | ${sc('San Francisco\n')} + ${ps('cities.txt.gz')} | ${sc('San Francisco\n')} + ${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt'} | ${sc('San Francisco\n')} + ${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt.gz'} | ${sc('San Francisco\n')} + `('readFile $filename', async ({ filename, expected }) => { const cspellIo = new CSpellIONode(); await expect(cspellIo.readFile(filename)).resolves.toEqual(expected); }); test.each` - filename | expected - ${__filename} | ${'Unhandled Request: fs:readFileSync' /* sc('This bit of text') */} - `('readFileSync', ({ filename, expected }) => { + filename | expected + ${ps('cities.not_found.txt')} | ${oc({ code: 'ENOENT' })} + ${ps('cities.not_found.txt.gz')} | ${oc({ code: 'ENOENT' })} + ${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.not_found.txt'} | ${oc({ code: 'ENOENT' })} + ${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/not_found/cities.txt.gz'} | ${oc({ code: 'ENOENT' })} + `('readFile not found $filename', async ({ filename, expected }) => { const cspellIo = new CSpellIONode(); - expect(() => cspellIo.readFileSync(filename)).toThrow(expected); + await expect(cspellIo.readFile(filename)).rejects.toEqual(expected); + }); + + test.each` + filename | expected + ${__filename} | ${sc('This bit of text')} + ${ps('cities.txt')} | ${sc('San Francisco\n')} + ${ps('cities.txt.gz')} | ${sc('San Francisco\n')} + `('readFileSync $filename', ({ filename, expected }) => { + const cspellIo = new CSpellIONode(); + expect(cspellIo.readFileSync(filename)).toEqual(expected); + }); + + const stats = { + urlA: { eTag: 'W/"10c5e3c7c73159515d4334813d6ba0255230270d92ebfdbd37151db7a0db5918"', mtimeMs: 0, size: -1 }, + urlB: { eTag: 'W/"10c5e3c7c73159515d4334813d6ba0255230270d92ebfdbd37151db7a0dbffff"', mtimeMs: 0, size: -1 }, + file1: { mtimeMs: 1658757408444.0342, size: 1886 }, + file2: { mtimeMs: 1658757408444.0342, size: 2886 }, + file3: { mtimeMs: 1758757408444.0342, size: 1886 }, + }; + + test.each` + left | right | expected + ${stats.urlA} | ${stats.urlA} | ${0} + ${stats.urlA} | ${stats.file1} | ${1} + ${stats.file1} | ${stats.file3} | ${-1} + ${stats.file2} | ${stats.file3} | ${1} + `('getStat $left <> $right', async ({ left, right, expected }) => { + const cspellIo = new CSpellIONode(); + const r = cspellIo.compareStats(left, right); + expect(r).toEqual(expected); }); - // readFile(_uriOrFilename: string): Promise { - // throw new ErrorNotImplemented('readFile'); - // } - // readFileSync(_uriOrFilename: string): string { - // throw new ErrorNotImplemented('readFileSync'); - // } // writeFile(_uriOrFilename: string, _content: string): Promise { // throw new ErrorNotImplemented('writeFile'); // } @@ -39,7 +72,4 @@ describe('CSpellIONode', () => { // getStatSync(_uriOrFilename: string): Stats { // throw new ErrorNotImplemented('getStatSync'); // } - // compareStats(left: Stats, right: Stats): number { - // return compareStats(left, right); - // } }); diff --git a/packages/cspell-io/src/CSpellIONode.ts b/packages/cspell-io/src/CSpellIONode.ts index bed5d2d3f86..2840d1ec496 100644 --- a/packages/cspell-io/src/CSpellIONode.ts +++ b/packages/cspell-io/src/CSpellIONode.ts @@ -5,7 +5,13 @@ import { ErrorNotImplemented } from './errors/ErrorNotImplemented'; import { registerHandlers } from './handlers/node/file'; import { Stats } from './models/Stats'; import { toURL } from './node/file/util'; -import { RequestFsReadFile, RequestFsReadFileSync } from './requests'; +import { + RequestFsReadFile, + RequestFsReadFileSync, + RequestFsStat, + RequestFsStatSync, + RequestFsWriteFile, +} from './requests'; export class CSpellIONode implements CSpellIO { constructor(readonly serviceBus = new ServiceBus()) { @@ -16,7 +22,7 @@ export class CSpellIONode implements CSpellIO { const url = toURL(uriOrFilename); const res = this.serviceBus.dispatch(RequestFsReadFile.create({ url })); if (!isServiceResponseSuccess(res)) { - throw res.error || new ErrorNotImplemented('readFile'); + throw genError(res.error, 'readFile'); } return res.value; } @@ -24,21 +30,39 @@ export class CSpellIONode implements CSpellIO { const url = toURL(uriOrFilename); const res = this.serviceBus.dispatch(RequestFsReadFileSync.create({ url })); if (!isServiceResponseSuccess(res)) { - throw res.error || new ErrorNotImplemented('readFileSync'); + throw genError(res.error, 'readFileSync'); } return res.value; - throw new ErrorNotImplemented('readFileSync'); } - writeFile(_uriOrFilename: string, _content: string): Promise { - throw new ErrorNotImplemented('writeFile'); + writeFile(uriOrFilename: string, _content: string): Promise { + const url = toURL(uriOrFilename); + const res = this.serviceBus.dispatch(RequestFsWriteFile.create({ url })); + if (!isServiceResponseSuccess(res)) { + throw genError(res.error, 'writeFile'); + } + return res.value; } - getStat(_uriOrFilename: string): Promise { - throw new ErrorNotImplemented('getStat'); + getStat(uriOrFilename: string): Promise { + const url = toURL(uriOrFilename); + const res = this.serviceBus.dispatch(RequestFsStat.create({ url })); + if (!isServiceResponseSuccess(res)) { + throw genError(res.error, 'getStat'); + } + return res.value; } - getStatSync(_uriOrFilename: string): Stats { - throw new ErrorNotImplemented('getStatSync'); + getStatSync(uriOrFilename: string): Stats { + const url = toURL(uriOrFilename); + const res = this.serviceBus.dispatch(RequestFsStatSync.create({ url })); + if (!isServiceResponseSuccess(res)) { + throw genError(res.error, 'getStatSync'); + } + return res.value; } compareStats(left: Stats, right: Stats): number { return compareStats(left, right); } } + +function genError(err: Error | undefined, alt: string): Error { + return err || new ErrorNotImplemented(alt); +} diff --git a/packages/cspell-io/src/handlers/node/file.ts b/packages/cspell-io/src/handlers/node/file.ts index dcac5ef63ab..2b01527164f 100644 --- a/packages/cspell-io/src/handlers/node/file.ts +++ b/packages/cspell-io/src/handlers/node/file.ts @@ -1,16 +1,22 @@ import { createRequestHandler, - ServiceBus, createResponse, - isServiceResponseSuccess, createResponseFail, isServiceResponseFailure, + isServiceResponseSuccess, + ServiceBus, } from '@cspell/cspell-service-bus'; -import { RequestFsReadBinaryFile, RequestFsReadFile, RequestZlibInflate } from '../../requests'; -import { promises as fs } from 'fs'; -import { deflateSync } from 'zlib'; -import { isZipped } from '../../node/file/util'; import assert from 'assert'; +import { promises as fs, readFileSync } from 'fs'; +import { gunzipSync } from 'zlib'; +import { fetchURL } from '../../node/file/fetch'; +import { + RequestFsReadBinaryFile, + RequestFsReadBinaryFileSync, + RequestFsReadFile, + RequestFsReadFileSync, + RequestZlibInflate, +} from '../../requests'; /** * Handle Binary File Reads @@ -22,6 +28,16 @@ const handleRequestFsReadBinaryFile = createRequestHandler( 'Node: Read Binary File.' ); +/** + * Handle Binary File Sync Reads + */ +const handleRequestFsReadBinaryFileSync = createRequestHandler( + RequestFsReadBinaryFileSync, + ({ params }) => createResponse(readFileSync(params.url)), + undefined, + 'Node: Sync Read Binary File.' +); + /** * Handle UTF-8 Text File Reads */ @@ -34,39 +50,60 @@ const handleRequestFsReadFile = createRequestHandler( assert(isServiceResponseFailure(res)); return createResponseFail(req, res.error); } - return createResponse(res.value.then((buf) => buf.toString('utf-8'))); + return createResponse(res.value.then((buf) => bufferToText(buf))); }, - RequestFsReadFile.type, + undefined, 'Node: Read Text File.' ); +/** + * Handle UTF-8 Text File Reads + */ +const handleRequestFsReadFileSync = createRequestHandler( + RequestFsReadFileSync, + (req, _, dispatcher) => { + const { url } = req.params; + const res = dispatcher.dispatch(RequestFsReadBinaryFileSync.create({ url })); + if (!isServiceResponseSuccess(res)) { + assert(isServiceResponseFailure(res)); + return createResponseFail(req, res.error); + } + return createResponse(bufferToText(res.value)); + }, + undefined, + 'Node: Sync Read Text File.' +); + /** * Handle deflating gzip data */ const handleRequestZlibInflate = createRequestHandler( RequestZlibInflate, - ({ params }) => createResponse(deflateSync(params.data).toString('utf-8')), - RequestZlibInflate.type, + ({ params }) => createResponse(gunzipSync(params.data).toString('utf-8')), + undefined, 'Node: gz deflate.' ); +const supportedFetchProtocols: Record = { 'http:': true, 'https:': true }; + /** * Handle reading gzip'ed text files. */ -const handleRequestFsReadFileGz = createRequestHandler( - RequestFsReadFile, - (req, next, dispatcher) => { +const handleRequestFsReadBinaryFileHttp = createRequestHandler( + RequestFsReadBinaryFile, + (req, next) => { const { url } = req.params; - if (!isZipped(url)) return next(req); - const result = dispatcher.dispatch(RequestFsReadBinaryFile.create({ url })); - return isServiceResponseSuccess(result) - ? createResponse(result.value.then((buf) => deflateSync(buf).toString('utf-8'))) - : result; + if (!(url.protocol in supportedFetchProtocols)) return next(req); + return createResponse(fetchURL(url)); }, undefined, - 'Node: Read GZ Text File.' + 'Node: Read Http(s) file.' ); +function bufferToText(buf: Buffer): string { + return buf[0] === 0x1f && buf[1] === 0x8b ? bufferToText(gunzipSync(buf)) : buf.toString('utf-8'); +} + export function registerHandlers(serviceBus: ServiceBus) { /** * Handlers are in order of low to high level @@ -74,9 +111,11 @@ export function registerHandlers(serviceBus: ServiceBus) { */ const handlers = [ handleRequestFsReadBinaryFile, + handleRequestFsReadBinaryFileSync, + handleRequestFsReadBinaryFileHttp, handleRequestFsReadFile, + handleRequestFsReadFileSync, handleRequestZlibInflate, - handleRequestFsReadFileGz, ]; handlers.forEach((handler) => serviceBus.addHandler(handler)); diff --git a/packages/cspell-io/src/node/file/fetch.ts b/packages/cspell-io/src/node/file/fetch.ts index 459149c6635..93e2271d551 100644 --- a/packages/cspell-io/src/node/file/fetch.ts +++ b/packages/cspell-io/src/node/file/fetch.ts @@ -1,5 +1,6 @@ import type { Headers } from 'node-fetch'; import nodeFetch from 'node-fetch'; +import { FetchUrlError } from './FetchError'; export const fetch = nodeFetch; @@ -7,3 +8,11 @@ export async function fetchHead(request: string | URL): Promise { const r = await fetch(request, { method: 'HEAD' }); return r.headers; } + +export async function fetchURL(url: URL): Promise { + const response = await fetch(url); + if (!response.ok) { + throw FetchUrlError.create(url, response.status); + } + return Buffer.from(await response.arrayBuffer()); +} diff --git a/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts b/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts index 2c1f4523316..654f8fa0b97 100644 --- a/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts +++ b/packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts @@ -5,3 +5,8 @@ interface RequestParams { readonly url: URL; } export const RequestFsReadBinaryFile = requestFactory>(RequestType); + +const RequestTypeSync = 'fs:readBinaryFileSync' as const; +export const RequestFsReadBinaryFileSync = requestFactory( + RequestTypeSync +); diff --git a/packages/cspell-io/src/requests/RequestFsWriteFile.ts b/packages/cspell-io/src/requests/RequestFsWriteFile.ts new file mode 100644 index 00000000000..84478669a7d --- /dev/null +++ b/packages/cspell-io/src/requests/RequestFsWriteFile.ts @@ -0,0 +1,7 @@ +import { requestFactory } from '@cspell/cspell-service-bus'; + +const RequestType = 'fs:writeFile' as const; +interface RequestParams { + readonly url: URL; +} +export const RequestFsWriteFile = requestFactory>(RequestType); diff --git a/packages/cspell-io/src/requests/index.ts b/packages/cspell-io/src/requests/index.ts index acdc4c29b1a..c15d1ae682b 100644 --- a/packages/cspell-io/src/requests/index.ts +++ b/packages/cspell-io/src/requests/index.ts @@ -1,5 +1,6 @@ -export { RequestFsReadBinaryFile } from './RequestFsReadBinaryFile'; +export { RequestFsReadBinaryFile, RequestFsReadBinaryFileSync } from './RequestFsReadBinaryFile'; export { RequestFsReadFile } from './RequestFsReadFile'; export { RequestFsReadFileSync } from './RequestFsReadFileSync'; export { RequestFsStat, RequestFsStatSync } from './RequestFsStat'; export { RequestZlibInflate } from './RequestZlibInflate'; +export { RequestFsWriteFile } from './RequestFsWriteFile';