Skip to content

Commit

Permalink
Support reading http files.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jason3S
Jason3S committed Jul 26, 2022
1 parent 70bf434 commit c527ac4
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 47 deletions.
62 changes: 46 additions & 16 deletions packages/cspell-io/src/CSpellIONode.test.ts
View file
@@ -1,6 +1,8 @@
import { CSpellIONode } from './CSpellIONode';
import { pathToSample as ps } from './test/helper';

const sc = expect.stringContaining;
const oc = expect.objectContaining;

describe('CSpellIONode', () => {
test('constructor', () => {
Expand All @@ -9,27 +11,58 @@ describe('CSpellIONode', () => {
});

test.each`
filename | expected
${__filename} | ${sc('This bit of text')}
`('readFile', async ({ filename, expected }) => {
filename | expected
${__filename} | ${sc('This bit of text')}
${ps('cities.txt')} | ${sc('San Francisco\n')}
${ps('cities.txt.gz')} | ${sc('San Francisco\n')}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt'} | ${sc('San Francisco\n')}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt.gz'} | ${sc('San Francisco\n')}
`('readFile $filename', async ({ filename, expected }) => {
const cspellIo = new CSpellIONode();
await expect(cspellIo.readFile(filename)).resolves.toEqual(expected);
});

test.each`
filename | expected
${__filename} | ${'Unhandled Request: fs:readFileSync' /* sc('This bit of text') */}
`('readFileSync', ({ filename, expected }) => {
filename | expected
${ps('cities.not_found.txt')} | ${oc({ code: 'ENOENT' })}
${ps('cities.not_found.txt.gz')} | ${oc({ code: 'ENOENT' })}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.not_found.txt'} | ${oc({ code: 'ENOENT' })}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/not_found/cities.txt.gz'} | ${oc({ code: 'ENOENT' })}
`('readFile not found $filename', async ({ filename, expected }) => {
const cspellIo = new CSpellIONode();
expect(() => cspellIo.readFileSync(filename)).toThrow(expected);
await expect(cspellIo.readFile(filename)).rejects.toEqual(expected);
});

test.each`
filename | expected
${__filename} | ${sc('This bit of text')}
${ps('cities.txt')} | ${sc('San Francisco\n')}
${ps('cities.txt.gz')} | ${sc('San Francisco\n')}
`('readFileSync $filename', ({ filename, expected }) => {
const cspellIo = new CSpellIONode();
expect(cspellIo.readFileSync(filename)).toEqual(expected);
});

const stats = {
urlA: { eTag: 'W/"10c5e3c7c73159515d4334813d6ba0255230270d92ebfdbd37151db7a0db5918"', mtimeMs: 0, size: -1 },
urlB: { eTag: 'W/"10c5e3c7c73159515d4334813d6ba0255230270d92ebfdbd37151db7a0dbffff"', mtimeMs: 0, size: -1 },
file1: { mtimeMs: 1658757408444.0342, size: 1886 },
file2: { mtimeMs: 1658757408444.0342, size: 2886 },
file3: { mtimeMs: 1758757408444.0342, size: 1886 },
};

test.each`
left | right | expected
${stats.urlA} | ${stats.urlA} | ${0}
${stats.urlA} | ${stats.file1} | ${1}
${stats.file1} | ${stats.file3} | ${-1}
${stats.file2} | ${stats.file3} | ${1}
`('getStat $left <> $right', async ({ left, right, expected }) => {
const cspellIo = new CSpellIONode();
const r = cspellIo.compareStats(left, right);
expect(r).toEqual(expected);
});

// readFile(_uriOrFilename: string): Promise<string> {
// throw new ErrorNotImplemented('readFile');
// }
// readFileSync(_uriOrFilename: string): string {
// throw new ErrorNotImplemented('readFileSync');
// }
// writeFile(_uriOrFilename: string, _content: string): Promise<void> {
// throw new ErrorNotImplemented('writeFile');
// }
Expand All @@ -39,7 +72,4 @@ describe('CSpellIONode', () => {
// getStatSync(_uriOrFilename: string): Stats {
// throw new ErrorNotImplemented('getStatSync');
// }
// compareStats(left: Stats, right: Stats): number {
// return compareStats(left, right);
// }
});
44 changes: 34 additions & 10 deletions packages/cspell-io/src/CSpellIONode.ts
View file
Expand Up @@ -5,7 +5,13 @@ import { ErrorNotImplemented } from './errors/ErrorNotImplemented';
import { registerHandlers } from './handlers/node/file';
import { Stats } from './models/Stats';
import { toURL } from './node/file/util';
import { RequestFsReadFile, RequestFsReadFileSync } from './requests';
import {
RequestFsReadFile,
RequestFsReadFileSync,
RequestFsStat,
RequestFsStatSync,
RequestFsWriteFile,
} from './requests';

export class CSpellIONode implements CSpellIO {
constructor(readonly serviceBus = new ServiceBus()) {
Expand All @@ -16,29 +22,47 @@ export class CSpellIONode implements CSpellIO {
const url = toURL(uriOrFilename);
const res = this.serviceBus.dispatch(RequestFsReadFile.create({ url }));
if (!isServiceResponseSuccess(res)) {
throw res.error || new ErrorNotImplemented('readFile');
throw genError(res.error, 'readFile');
}
return res.value;
}
readFileSync(uriOrFilename: string): string {
const url = toURL(uriOrFilename);
const res = this.serviceBus.dispatch(RequestFsReadFileSync.create({ url }));
if (!isServiceResponseSuccess(res)) {
throw res.error || new ErrorNotImplemented('readFileSync');
throw genError(res.error, 'readFileSync');
}
return res.value;
throw new ErrorNotImplemented('readFileSync');
}
writeFile(_uriOrFilename: string, _content: string): Promise<void> {
throw new ErrorNotImplemented('writeFile');
writeFile(uriOrFilename: string, _content: string): Promise<void> {
const url = toURL(uriOrFilename);
const res = this.serviceBus.dispatch(RequestFsWriteFile.create({ url }));
if (!isServiceResponseSuccess(res)) {
throw genError(res.error, 'writeFile');
}
return res.value;
}
getStat(_uriOrFilename: string): Promise<Stats> {
throw new ErrorNotImplemented('getStat');
getStat(uriOrFilename: string): Promise<Stats> {
const url = toURL(uriOrFilename);
const res = this.serviceBus.dispatch(RequestFsStat.create({ url }));
if (!isServiceResponseSuccess(res)) {
throw genError(res.error, 'getStat');
}
return res.value;
}
getStatSync(_uriOrFilename: string): Stats {
throw new ErrorNotImplemented('getStatSync');
getStatSync(uriOrFilename: string): Stats {
const url = toURL(uriOrFilename);
const res = this.serviceBus.dispatch(RequestFsStatSync.create({ url }));
if (!isServiceResponseSuccess(res)) {
throw genError(res.error, 'getStatSync');
}
return res.value;
}
compareStats(left: Stats, right: Stats): number {
return compareStats(left, right);
}
}

function genError(err: Error | undefined, alt: string): Error {
return err || new ErrorNotImplemented(alt);
}
79 changes: 59 additions & 20 deletions packages/cspell-io/src/handlers/node/file.ts
View file
@@ -1,16 +1,22 @@
import {
createRequestHandler,
ServiceBus,
createResponse,
isServiceResponseSuccess,
createResponseFail,
isServiceResponseFailure,
isServiceResponseSuccess,
ServiceBus,
} from '@cspell/cspell-service-bus';
import { RequestFsReadBinaryFile, RequestFsReadFile, RequestZlibInflate } from '../../requests';
import { promises as fs } from 'fs';
import { deflateSync } from 'zlib';
import { isZipped } from '../../node/file/util';
import assert from 'assert';
import { promises as fs, readFileSync } from 'fs';
import { gunzipSync } from 'zlib';
import { fetchURL } from '../../node/file/fetch';
import {
RequestFsReadBinaryFile,
RequestFsReadBinaryFileSync,
RequestFsReadFile,
RequestFsReadFileSync,
RequestZlibInflate,
} from '../../requests';

/**
* Handle Binary File Reads
Expand All @@ -22,6 +28,16 @@ const handleRequestFsReadBinaryFile = createRequestHandler(
'Node: Read Binary File.'
);

/**
* Handle Binary File Sync Reads
*/
const handleRequestFsReadBinaryFileSync = createRequestHandler(
RequestFsReadBinaryFileSync,
({ params }) => createResponse(readFileSync(params.url)),
undefined,
'Node: Sync Read Binary File.'
);

/**
* Handle UTF-8 Text File Reads
*/
Expand All @@ -34,49 +50,72 @@ const handleRequestFsReadFile = createRequestHandler(
assert(isServiceResponseFailure(res));
return createResponseFail(req, res.error);
}
return createResponse(res.value.then((buf) => buf.toString('utf-8')));
return createResponse(res.value.then((buf) => bufferToText(buf)));
},
RequestFsReadFile.type,
undefined,
'Node: Read Text File.'
);

/**
* Handle UTF-8 Text File Reads
*/
const handleRequestFsReadFileSync = createRequestHandler(
RequestFsReadFileSync,
(req, _, dispatcher) => {
const { url } = req.params;
const res = dispatcher.dispatch(RequestFsReadBinaryFileSync.create({ url }));
if (!isServiceResponseSuccess(res)) {
assert(isServiceResponseFailure(res));
return createResponseFail(req, res.error);
}
return createResponse(bufferToText(res.value));
},
undefined,
'Node: Sync Read Text File.'
);

/**
* Handle deflating gzip data
*/
const handleRequestZlibInflate = createRequestHandler(
RequestZlibInflate,
({ params }) => createResponse(deflateSync(params.data).toString('utf-8')),
RequestZlibInflate.type,
({ params }) => createResponse(gunzipSync(params.data).toString('utf-8')),
undefined,
'Node: gz deflate.'
);

const supportedFetchProtocols: Record<string, true | undefined> = { 'http:': true, 'https:': true };

/**
* Handle reading gzip'ed text files.
*/
const handleRequestFsReadFileGz = createRequestHandler(
RequestFsReadFile,
(req, next, dispatcher) => {
const handleRequestFsReadBinaryFileHttp = createRequestHandler(
RequestFsReadBinaryFile,
(req, next) => {
const { url } = req.params;
if (!isZipped(url)) return next(req);
const result = dispatcher.dispatch(RequestFsReadBinaryFile.create({ url }));
return isServiceResponseSuccess(result)
? createResponse(result.value.then((buf) => deflateSync(buf).toString('utf-8')))
: result;
if (!(url.protocol in supportedFetchProtocols)) return next(req);
return createResponse(fetchURL(url));
},
undefined,
'Node: Read GZ Text File.'
'Node: Read Http(s) file.'
);

function bufferToText(buf: Buffer): string {
return buf[0] === 0x1f && buf[1] === 0x8b ? bufferToText(gunzipSync(buf)) : buf.toString('utf-8');
}

export function registerHandlers(serviceBus: ServiceBus) {
/**
* Handlers are in order of low to high level
* Order is VERY important.
*/
const handlers = [
handleRequestFsReadBinaryFile,
handleRequestFsReadBinaryFileSync,
handleRequestFsReadBinaryFileHttp,
handleRequestFsReadFile,
handleRequestFsReadFileSync,
handleRequestZlibInflate,
handleRequestFsReadFileGz,
];

handlers.forEach((handler) => serviceBus.addHandler(handler));
Expand Down
9 changes: 9 additions & 0 deletions packages/cspell-io/src/node/file/fetch.ts
View file
@@ -1,9 +1,18 @@
import type { Headers } from 'node-fetch';
import nodeFetch from 'node-fetch';
import { FetchUrlError } from './FetchError';

export const fetch = nodeFetch;

export async function fetchHead(request: string | URL): Promise<Headers> {
const r = await fetch(request, { method: 'HEAD' });
return r.headers;
}

export async function fetchURL(url: URL): Promise<Buffer> {
const response = await fetch(url);
if (!response.ok) {
throw FetchUrlError.create(url, response.status);
}
return Buffer.from(await response.arrayBuffer());
}
5 changes: 5 additions & 0 deletions packages/cspell-io/src/requests/RequestFsReadBinaryFile.ts
View file
Expand Up @@ -5,3 +5,8 @@ interface RequestParams {
readonly url: URL;
}
export const RequestFsReadBinaryFile = requestFactory<typeof RequestType, RequestParams, Promise<Buffer>>(RequestType);

const RequestTypeSync = 'fs:readBinaryFileSync' as const;
export const RequestFsReadBinaryFileSync = requestFactory<typeof RequestTypeSync, RequestParams, Buffer>(
RequestTypeSync
);
7 changes: 7 additions & 0 deletions packages/cspell-io/src/requests/RequestFsWriteFile.ts
View file
@@ -0,0 +1,7 @@
import { requestFactory } from '@cspell/cspell-service-bus';

const RequestType = 'fs:writeFile' as const;
interface RequestParams {
readonly url: URL;
}
export const RequestFsWriteFile = requestFactory<typeof RequestType, RequestParams, Promise<void>>(RequestType);
3 changes: 2 additions & 1 deletion packages/cspell-io/src/requests/index.ts
View file
@@ -1,5 +1,6 @@
export { RequestFsReadBinaryFile } from './RequestFsReadBinaryFile';
export { RequestFsReadBinaryFile, RequestFsReadBinaryFileSync } from './RequestFsReadBinaryFile';
export { RequestFsReadFile } from './RequestFsReadFile';
export { RequestFsReadFileSync } from './RequestFsReadFileSync';
export { RequestFsStat, RequestFsStatSync } from './RequestFsStat';
export { RequestZlibInflate } from './RequestZlibInflate';
export { RequestFsWriteFile } from './RequestFsWriteFile';

0 comments on commit c527ac4

Please sign in to comment.