dev: CSpell IO layer (#3310)

* dev: CSpell IO layer
* fix: Add encoding to reads
* fix: add `data:` url encoder

packages/cspell-io/src/CSpellIO.ts

@@ -1,8 +1,9 @@
+import { BufferEncoding } from './models/BufferEncoding';
 import type { Stats } from './models';
 
 export interface CSpellIO {
-    readFile(uriOrFilename: string): Promise<string>;
-    readFileSync(uriOrFilename: string): string;
+    readFile(uriOrFilename: string, encoding?: BufferEncoding): Promise<string>;
+    readFileSync(uriOrFilename: string, encoding?: BufferEncoding): string;
     writeFile(uriOrFilename: string, content: string): Promise<void>;
     getStat(uriOrFilename: string): Promise<Stats>;
     getStatSync(uriOrFilename: string): Stats;

packages/cspell-io/src/CSpellIONode.ts

@@ -18,17 +18,17 @@ export class CSpellIONode implements CSpellIO {
         registerHandlers(serviceBus);
     }
 
-    readFile(uriOrFilename: string): Promise<string> {
+    readFile(uriOrFilename: string, encoding: BufferEncoding = 'utf8'): Promise<string> {
         const url = toURL(uriOrFilename);
-        const res = this.serviceBus.dispatch(RequestFsReadFile.create({ url }));
+        const res = this.serviceBus.dispatch(RequestFsReadFile.create({ url, encoding }));
         if (!isServiceResponseSuccess(res)) {
             throw genError(res.error, 'readFile');
         }
         return res.value;
     }
-    readFileSync(uriOrFilename: string): string {
+    readFileSync(uriOrFilename: string, encoding: BufferEncoding = 'utf8'): string {
         const url = toURL(uriOrFilename);
-        const res = this.serviceBus.dispatch(RequestFsReadFileSync.create({ url }));
+        const res = this.serviceBus.dispatch(RequestFsReadFileSync.create({ url, encoding }));
         if (!isServiceResponseSuccess(res)) {
             throw genError(res.error, 'readFileSync');
         }

packages/cspell-io/src/__snapshots__/index.test.ts.snap

@@ -0,0 +1,17 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`index api 1`] = `
+Array [
+  "CSpellIONode => function",
+  "asyncIterableToArray => function",
+  "encodeDataUrl => function",
+  "getStat => function",
+  "getStatSync => function",
+  "readFile => function",
+  "readFileSync => function",
+  "toDataUrl => function",
+  "writeToFile => function",
+  "writeToFileIterable => function",
+  "writeToFileIterableP => function",
+]
+`;

packages/cspell-io/src/errors/error.ts

@@ -0,0 +1,8 @@
+export function toError(e: unknown): Error {
+    if (e instanceof Error) return e;
+    if (typeof e === 'object' && e && typeof (e as Error).message === 'string') {
+        return e as Error;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return Error(e && (e as any).toString());
+}

packages/cspell-io/src/errors/index.ts

@@ -1,9 +1,2 @@
-import { format } from 'util';
-
-export function toError(e: unknown): Error {
-    if (e instanceof Error) return e;
-    if (typeof e === 'object' && e && (e as Error).message) {
-        return e as Error;
-    }
-    return Error(format(e));
-}
+export { toError } from './error';
+export { ErrorNotImplemented } from './ErrorNotImplemented';

packages/cspell-io/src/handlers/node/file.ts

@@ -55,13 +55,13 @@ const handleRequestFsReadBinaryFileSync = createRequestHandler(
 const handleRequestFsReadFile = createRequestHandler(
     RequestFsReadFile,
     (req, _, dispatcher) => {
-        const { url } = req.params;
+        const { url, encoding } = req.params;
         const res = dispatcher.dispatch(RequestFsReadBinaryFile.create({ url }));
         if (!isServiceResponseSuccess(res)) {
             assert(isServiceResponseFailure(res));
             return createResponseFail(req, res.error);
         }
-        return createResponse(res.value.then((buf) => bufferToText(buf)));
+        return createResponse(res.value.then((buf) => bufferToText(buf, encoding)));
     },
     undefined,
     'Node: Read Text File.'
@@ -73,13 +73,13 @@ const handleRequestFsReadFile = createRequestHandler(
 const handleRequestFsReadFileSync = createRequestHandler(
     RequestFsReadFileSync,
     (req, _, dispatcher) => {
-        const { url } = req.params;
+        const { url, encoding } = req.params;
         const res = dispatcher.dispatch(RequestFsReadBinaryFileSync.create({ url }));
         if (!isServiceResponseSuccess(res)) {
             assert(isServiceResponseFailure(res));
             return createResponseFail(req, res.error);
         }
-        return createResponse(bufferToText(res.value));
+        return createResponse(bufferToText(res.value, encoding));
     },
     undefined,
     'Node: Sync Read Text File.'
@@ -111,8 +111,8 @@ const handleRequestFsReadBinaryFileHttp = createRequestHandler(
     'Node: Read Http(s) file.'
 );
 
-function bufferToText(buf: Buffer): string {
-    return buf[0] === 0x1f && buf[1] === 0x8b ? bufferToText(gunzipSync(buf)) : buf.toString('utf-8');
+function bufferToText(buf: Buffer, encoding: BufferEncoding): string {
+    return buf[0] === 0x1f && buf[1] === 0x8b ? bufferToText(gunzipSync(buf), encoding) : buf.toString(encoding);
 }
 
 /**

packages/cspell-io/src/index.test.ts

@@ -4,4 +4,11 @@ describe('index', () => {
     test('exports', () => {
         expect(index.readFile).toBeDefined();
     });
+
+    test('api', () => {
+        const api = Object.entries(index)
+            .map(([key, value]) => `${key} => ${typeof value}`)
+            .sort();
+        expect(api).toMatchSnapshot();
+    });
 });

packages/cspell-io/src/index.ts

@@ -11,3 +11,4 @@ export {
 export type { Stats } from './models/Stats';
 export type { CSpellIO } from './CSpellIO';
 export { CSpellIONode } from './CSpellIONode';
+export { encodeDataUrl, toDataUrl } from './node/dataUrl';

packages/cspell-io/src/models/BufferEncoding.ts

@@ -0,0 +1,11 @@
+export type BufferEncoding =
+    | 'ascii'
+    | 'utf8'
+    | 'utf-8'
+    | 'utf16le'
+    | 'ucs2'
+    | 'ucs-2'
+    | 'base64'
+    | 'base64url'
+    | 'latin1'
+    | 'hex';

packages/cspell-io/src/node/dataUrl.test.ts

@@ -0,0 +1,30 @@
+import { toDataUrl, encodeDataUrl, decodeDataUrl } from './dataUrl';
+
+describe('dataUrl', () => {
+    test.each`
+        data                            | mediaType                   | attributes                     | expected
+        ${'Hello, World!'}              | ${'text/plain'}             | ${undefined}                   | ${'data:text/plain;charset=utf8,Hello%2C%20World!'}
+        ${'Hello, World!'}              | ${'text/plain'}             | ${[['filename', 'hello.txt']]} | ${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'}
+        ${'Hello, World! %%%%$$$$,,,,'} | ${'text/plain'}             | ${undefined}                   | ${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'}
+        ${Buffer.from('Hello, World!')} | ${'text/plain'}             | ${[['filename', 'hello.txt']]} | ${'data:text/plain;filename=hello.txt;base64,SGVsbG8sIFdvcmxkIQ' /* cspell:disable-line */}
+        ${'☸☹☺☻☼☾☿'}                    | ${'text/plain'}             | ${undefined}                   | ${'data:text/plain;charset=utf8;base64,4pi44pi54pi64pi74pi84pi-4pi_'}
+        ${'Hello, World!'}              | ${'application/vnd.cspell'} | ${undefined}                   | ${'data:application/vnd.cspell;charset=utf8,Hello%2C%20World!'}
+    `('encodeDataUrl $data', ({ data, mediaType, attributes, expected }) => {
+        const url = encodeDataUrl(data, mediaType, attributes);
+        expect(url).toEqual(expected);
+        const urlObj = toDataUrl(data, mediaType, attributes);
+        expect(urlObj.toString()).toEqual(url);
+    });
+
+    test.each`
+        url                                                                          | expected
+        ${'data:text/plain;charset=utf8,Hello%2C%20World!'}                          | ${{ mediaType: 'text/plain', encoding: 'utf8', data: Buffer.from('Hello, World!'), attributes: new Map([['charset', 'utf8']]) }}
+        ${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'}       | ${{ mediaType: 'text/plain', encoding: 'utf8', data: Buffer.from('Hello, World!'), attributes: new Map([['charset', 'utf8'], ['filename', 'hello.txt']]) }}
+        ${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'} | ${{ mediaType: 'text/plain', encoding: 'utf8', data: Buffer.from('Hello, World! %%%%$$$$,,,,'), attributes: new Map([['charset', 'utf8']]) }}
+        ${'data:text/plain;base64,SGVsbG8sIFdvcmxkIQ' /* cspell:disable-line */}     | ${{ mediaType: 'text/plain', data: Buffer.from('Hello, World!'), attributes: new Map() }}
+        ${'data:text/plain;charset=utf8;base64,4pi44pi54pi64pi74pi84pi-4pi_'}        | ${{ mediaType: 'text/plain', encoding: 'utf8', data: Buffer.from('☸☹☺☻☼☾☿'), attributes: new Map([['charset', 'utf8']]) }}
+    `('encodeDataUrl $url', ({ url, expected }) => {
+        const data = decodeDataUrl(url);
+        expect(data).toEqual(expected);
+    });
+});

packages/cspell-io/src/node/dataUrl.ts

@@ -0,0 +1,76 @@
+/**
+ * Generates a string of the following format:
+ *
+ * `data:[mediaType][;charset=<encoding>[;base64],<data>`
+ *
+ * - `encoding` - defaults to `utf8` for text data
+ * @param data
+ * @param mediaType - The mediaType is a [MIME](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types) type string
+ * @param attributes - Additional attributes
+ */
+export function encodeDataUrl(
+    data: string | Buffer,
+    mediaType: string,
+    attributes?: Iterable<readonly [string, string]> | undefined
+): string {
+    if (typeof data === 'string') return encodeString(data, mediaType, attributes);
+    const attribs = encodeAttributes(attributes || []);
+    return `data:${mediaType}${attribs};base64,${data.toString('base64url')}`;
+}
+
+export function toDataUrl(
+    data: string | Buffer,
+    mediaType: string,
+    attributes?: Iterable<[string, string]> | undefined
+): URL {
+    return new URL(encodeDataUrl(data, mediaType, attributes));
+}
+
+function encodeString(
+    data: string,
+    mediaType: string | undefined,
+    attributes: Iterable<readonly [string, string]> | undefined
+): string {
+    mediaType = mediaType || 'text/plain';
+    attributes = attributes || [];
+    const asUrlComp = encodeURIComponent(data);
+    const asBase64 = Buffer.from(data).toString('base64url');
+    const useBase64 = asBase64.length < asUrlComp.length - 7;
+    const encoded = useBase64 ? asBase64 : asUrlComp;
+    // Ensure charset is first.
+    const attribMap = new Map([['charset', 'utf8'] as readonly [string, string]].concat([...attributes]));
+    attribMap.set('charset', 'utf8'); // Make sure it is always `utf8`.
+    const attribs = encodeAttributes(attribMap);
+    return `data:${mediaType}${attribs}${useBase64 ? ';base64' : ''},${encoded}`;
+}
+
+export interface DecodedDataUrl {
+    data: Buffer;
+    mediaType: string;
+    encoding?: string | undefined;
+    attributes: Map<string, string>;
+}
+
+function encodeAttributes(attributes: Iterable<readonly [string, string]>): string {
+    return [...attributes].map(([key, value]) => `;${key}=${encodeURIComponent(value)}`).join('');
+}
+
+const dataUrlRegExHead = /^data:(?<mediaType>[^;,]*)(?<attributes>(?:;[^=]+=[^;,]*)*)(?<base64>;base64)?$/;
+
+export function decodeDataUrl(url: string): DecodedDataUrl {
+    const [head, encodedData] = url.split(',', 2);
+    if (!head || encodedData === undefined) throw Error('Not a data url');
+    const match = head.match(dataUrlRegExHead);
+    if (!match || !match.groups) throw Error('Not a data url');
+    const mediaType = match.groups['mediaType'] || '';
+    const rawAttributes = (match.groups['attributes'] || '')
+        .split(';')
+        .filter((a) => !!a)
+        .map((entry) => entry.split('=', 2))
+        .map(([key, value]) => [key, decodeURIComponent(value)] as [string, string]);
+    const attributes = new Map(rawAttributes);
+    const encoding = attributes.get('charset');
+    const isBase64 = !!match.groups['base64'];
+    const data = isBase64 ? Buffer.from(encodedData, 'base64url') : Buffer.from(decodeURIComponent(encodedData));
+    return { mediaType, data, encoding, attributes };
+}

packages/cspell-io/src/requests/RequestFsReadFile.ts

@@ -3,5 +3,6 @@ import { requestFactory } from '@cspell/cspell-service-bus';
 const RequestType = 'fs:readFile' as const;
 interface RequestParams {
     readonly url: URL;
+    readonly encoding: BufferEncoding;
 }
 export const RequestFsReadFile = requestFactory<typeof RequestType, RequestParams, Promise<string>>(RequestType);

packages/cspell-io/src/requests/RequestFsReadFileSync.ts

@@ -3,5 +3,6 @@ import { requestFactory } from '@cspell/cspell-service-bus';
 const RequestType = 'fs:readFileSync' as const;
 interface RequestParams {
     readonly url: URL;
+    readonly encoding: BufferEncoding;
 }
 export const RequestFsReadFileSync = requestFactory<typeof RequestType, RequestParams, string>(RequestType);