Targz (#121)

* add wip broken monstrosity targz

* better, but still super idk man

* uh, this actually made a file that winrar could open

although the internal filepath was based on the full path from c:\ but this is promising. I still think things are done idiomatically wrong though.

* add directory test

and fixes

* work off base directory resolution

* add atomic shortcuts and tests

* add more unit tests

* fix type in Untar?

* add gzip methods

* I think this should be more efficient

* simplify Tar

* fix imports

* utilize scan in targz

* add some more types

* lint

* utilize _read

* coverage fixes

* remove state

* reorg

* obvious missed type, doesn't really effect anything though

* so long as circulars aren't a problem, this should reduce dupe

* fix failing test

* better stage names

* rewrite Untar and make progress

* fix some names

* still not correct, but probably moreso

* added some state, and now it works

but I don't think it should require state to do this

* remove extra state

* more tests, but trouble in checksum valley

* it wasn't the checksum, it was empty directories

now other tests are running, and other problems

* fix?

* add a test, yolo

* coverage adjustments

package.json

@@ -36,13 +36,13 @@
     "node": ">=10.1.0"
   },
   "devDependencies": {
-    "@types/node": "^11.9.6",
+    "@types/node": "^11.11.0",
     "ava": "^1.0.1",
     "eslint": "^5.2.0",
-    "eslint-config-klasa": "github:dirigeants/klasa-lint",
+    "eslint-config-klasa": "dirigeants/klasa-lint",
     "nyc": "^13.0.1",
-    "tslint": "^5.7.0",
     "ts-node": "^8.0.3",
+    "tslint": "^5.7.0",
     "typedoc": "^0.14.2",
     "typescript": "^3.0.1"
   },

scripts/.azure-test-template.yml

@@ -20,8 +20,11 @@ jobs:
         versionSpec: $(nodeVersion)
       displayName: 'Install Node.js'
     - script: 'yarn'
+      displayName: 'Install Dependencies'
     - script: 'yarn run test:coverage'
+      displayName: 'Run Tests'
     - script: 'yarn run coverage:azure'
+      displayName: 'Generate Reports'
     - task: PublishCodeCoverageResults@1
       inputs:
         codeCoverageTool: 'cobertura'

src/index.ts

@@ -21,6 +21,10 @@ export { default as ensureLink } from './nextra/createLink';
 export { default as ensureLinkAtomic } from './nextra/createLinkAtomic';
 export { default as ensureSymlink } from './nextra/createSymlink';
 export { default as ensureSymlinkAtomic } from './nextra/createSymlinkAtomic';
+export { default as gunzip } from './nextra/gunzip';
+export { default as gunzipAtomic } from './nextra/gunzipAtomic';
+export { default as gzip } from './nextra/gzip';
+export { default as gzipAtomic } from './nextra/gzip';
 export { default as linkAtomic } from './nextra/linkAtomic';
 export { default as mkdirp } from './nextra/mkdirs';
 export { default as mkdirs } from './nextra/mkdirs';
@@ -37,6 +41,10 @@ export { default as readJson } from './nextra/readJSON';
 export { default as remove } from './nextra/remove';
 export { default as scan } from './nextra/scan';
 export { default as symlinkAtomic } from './nextra/symlinkAtomic';
+export { default as targz } from './nextra/targz';
+export { default as targzAtomic } from './nextra/targzAtomic';
+export { default as unTargz } from './nextra/unTargz';
+export { default as unTargzAtomic } from './nextra/unTargzAtomic';
 export { default as writeFileAtomic } from './nextra/writeFileAtomic';
 export { default as writeJSON } from './nextra/writeJSON';
 export { default as writeJSONAtomic } from './nextra/writeJSONAtomic';

src/nextra/copy.ts

@@ -1,6 +1,6 @@
 import { resolve, dirname, join, basename } from 'path';
 
-import { replaceEsc, isSrcKid } from '../util';
+import { replaceEsc, isSrcKid } from '../utils/util';
 import { access, readlink, mkdir, symlink, copyFile, lstat, stat, chmod, readdir, Stats } from '../fs';
 
 import mkdirs from './mkdirs';

src/nextra/copyFileAtomic.ts

@@ -1,4 +1,4 @@
-import { tempFile } from '../util';
+import { tempFile } from '../utils/util';
 import { copyFile } from '../fs';
 
 import move from './move';

src/nextra/gunzip.ts

@@ -0,0 +1,23 @@
+import { createGunzip } from 'zlib';
+
+import { createWriteStream, createReadStream } from '../fs';
+import { pipelinePromise } from '../utils/util';
+import gunzipAtomic from './gunzipAtomic';
+
+/**
+ * Un-Gzips a file
+ * @function gunzip
+ * @memberof fsn/nextra
+ * @param fileName The filename of the output file
+ * @param inputFile The filepath of the archive
+ * @param atomic If the unzip file should be created atomically
+ */
+export default async function gzip(fileName: string, inputFile: string, atomic: boolean = false): Promise<void> {
+	if (atomic) return gunzipAtomic(fileName, inputFile);
+
+	return pipelinePromise(
+		createReadStream(inputFile),
+		createGunzip(),
+		createWriteStream(fileName)
+	);
+}

src/nextra/gunzipAtomic.ts

@@ -0,0 +1,16 @@
+import gunzip from './gunzip';
+import move from './move';
+import { tempFile } from '../utils/util';
+
+/**
+ * Un-Gzips a file atomically.
+ * @function gunzipAtomic
+ * @memberof fsn/nextra
+ * @param fileName The filename of the output file
+ * @param inputFile The filepath of the archive
+ */
+export default async function gzipAtomic(fileName: string, inputFile: string): Promise<void> {
+	const tempPath = tempFile();
+	await gunzip(tempPath, inputFile);
+	return move(tempPath, fileName, { overwrite: true });
+}

src/nextra/gzip.ts

@@ -0,0 +1,23 @@
+import { createGzip } from 'zlib';
+
+import { createWriteStream, createReadStream } from '../fs';
+import { pipelinePromise } from '../utils/util';
+import gzipAtomic from './gzipAtomic';
+
+/**
+ * Gzips a file
+ * @function gzip
+ * @memberof fsn/nextra
+ * @param fileName The filename of the archive
+ * @param inputFile The filepath of the input file
+ * @param atomic If the gzip file should be created
+ */
+export default async function gzip(fileName: string, inputFile: string, atomic: boolean = false): Promise<void> {
+	if (atomic) return gzipAtomic(fileName, inputFile);
+
+	return pipelinePromise(
+		createReadStream(inputFile),
+		createGzip(),
+		createWriteStream(fileName)
+	);
+}

src/nextra/gzipAtomic.ts

@@ -0,0 +1,16 @@
+import gzip from './gzip';
+import move from './move';
+import { tempFile } from '../utils/util';
+
+/**
+ * Gzips a file atomically.
+ * @function gzipAtomic
+ * @memberof fsn/nextra
+ * @param fileName The filename of the archive
+ * @param inputFile The filepath of the input file
+ */
+export default async function gzipAtomic(fileName: string, inputFile: string): Promise<void> {
+	const tempPath = tempFile();
+	await gzip(tempPath, inputFile);
+	return move(tempPath, fileName, { overwrite: true });
+}

src/nextra/linkAtomic.ts

@@ -1,4 +1,4 @@
-import { tempFile } from '../util';
+import { tempFile } from '../utils/util';
 import { link } from '../fs';
 
 import move from './move';

src/nextra/mkdirs.ts

@@ -1,6 +1,6 @@
 import { resolve, dirname, normalize, sep } from 'path';
 
-import { isWindows } from '../util';
+import { isWindows } from '../utils/util';
 import { stat, mkdir } from '../fs';
 
 /**

src/nextra/move.ts

@@ -1,6 +1,6 @@
 import { resolve, dirname } from 'path';
 
-import { isSrcKid } from '../util';
+import { isSrcKid } from '../utils/util';
 import { access, rename, stat } from '../fs';
 
 import remove from './remove';

src/nextra/remove.ts

@@ -1,6 +1,6 @@
 import { join } from 'path';
 
-import { isWindows, setTimeoutPromise } from '../util';
+import { isWindows, setTimeoutPromise } from '../utils/util';
 import { lstat, unlink, rmdir, chmod, readdir } from '../fs';
 
 /**

src/nextra/symlinkAtomic.ts

@@ -1,4 +1,4 @@
-import { tempFile } from '../util';
+import { tempFile } from '../utils/util';
 import { symlink } from '../fs';
 import { SymLinkType } from './createSymlink';
 

src/nextra/targz.ts

@@ -0,0 +1,35 @@
+import { createGzip } from 'zlib';
+
+import { createWriteStream } from '../fs';
+import Tar from '../utils/Tar';
+import { pipelinePromise } from '../utils/util';
+import { dirname } from 'path';
+import scan from './scan';
+import targzAtomic from './targzAtomic';
+
+
+/**
+ * Tar/Gzips a directory or array of files.
+ * @function targz
+ * @memberof fsn/nextra
+ * @param fileName The filename of the archive
+ * @param inputFiles The directory or array of filepaths to .tar.gz
+ * @param options The options for this .tar.gz
+ */
+export default async function targz(fileName: string, inputFiles: string | string[], atomic: boolean = false): Promise<void> {
+	if (atomic) return targzAtomic(fileName, inputFiles);
+	if (!Array.isArray(inputFiles)) inputFiles = [inputFiles];
+
+	const tar = new Tar(dirname(inputFiles[0]));
+
+	for (const input of inputFiles) {
+		const files = await scan(input, { filter: stats => stats.isFile() });
+		for (const [file, stats] of files) tar.append(file, stats);
+	}
+
+	return pipelinePromise(
+		tar,
+		createGzip(),
+		createWriteStream(fileName)
+	);
+}

src/nextra/targzAtomic.ts

@@ -0,0 +1,16 @@
+import targz from './targz';
+import move from './move';
+import { tempFile } from '../utils/util';
+
+/**
+ * Tar/Gzips a directory or array of files.
+ * @function targzAtomic
+ * @memberof fsn/nextra
+ * @param fileName The filename of the archive
+ * @param inputFiles The directory or array of filepaths to .tar.gz
+ */
+export default async function targzAtomic(fileName: string, inputFiles: string | string[]): Promise<void> {
+	const tempPath = tempFile();
+	await targz(tempPath, inputFiles);
+	return move(tempPath, fileName, { overwrite: true });
+}

src/nextra/unTargz.ts

@@ -0,0 +1,23 @@
+import { createGunzip } from 'zlib';
+import { resolve } from 'path';
+
+import { createReadStream } from '../fs';
+import Untar from '../utils/Untar';
+import outputFile from './outputFile';
+import outputFileAtomic from './outputFileAtomic';
+
+
+/**
+ * Extracts files from .tar.gz archives.
+ * @function unTargz
+ * @memberof fsn/nextra
+ * @param outputDirectory The directory to extract to
+ * @param inputFile The archive file
+ * @param atomic The if the writes should be atomic
+ */
+export default async function unTargz(outputDirectory: string, inputFile: string, atomic: boolean = false): Promise<void> {
+	const tar = createReadStream(inputFile).pipe(createGunzip()).pipe(new Untar());
+	const writeMethod = atomic ? outputFile : outputFileAtomic;
+
+	for await (const { header, file } of tar.files()) await writeMethod(resolve(outputDirectory, header.filename), file);
+}

src/nextra/unTargzAtomic.ts

@@ -0,0 +1,14 @@
+import unTargz from './unTargz';
+
+
+/**
+ * Extracts files from .tar.gz archives and writes them atomically.
+ * @function unTargzAtomic
+ * @memberof fsn/nextra
+ * @param outputDirectory The directory to extract to
+ * @param inputFile The archive file
+ * @param atomic The if the writes should be atomic
+ */
+export default async function unTargzAtomic(outputDirectory: string, inputFile: string): Promise<void> {
+	return unTargz(outputDirectory, inputFile, true);
+}

src/nextra/writeFileAtomic.ts

@@ -1,4 +1,4 @@
-import { tempFile } from '../util';
+import { tempFile } from '../utils/util';
 import { writeFile } from '../fs';
 
 import move from './move';

src/utils/Tar.ts

@@ -0,0 +1,64 @@
+import { relative } from 'path';
+import { Readable } from 'stream';
+import { encodeHeader } from './header';
+import { createReadStream, Stats } from '../fs';
+
+export default class Tar extends Readable {
+
+	private base: string;
+	private written: number = 0;
+	private recordSize = 512;
+	private queue: Array<{ header: Buffer, file: Readable, size: number }> = [];
+
+	public constructor(base: string) {
+		super();
+
+		this.base = base;
+	}
+
+	public async _read(): Promise<void> {
+		if (!this.queue.length) {
+			this.push(null);
+			return;
+		}
+
+		const { header, file, size } = this.queue.shift();
+		const { written } = this;
+
+		this.written += header.length;
+
+		const fileChunks = [];
+
+		for await (const chunk of file) {
+			fileChunks.push(chunk);
+			this.written += chunk.length;
+		}
+
+		// Hard to produce, requires a size perfectibly divisible by the recordSize
+		/* istanbul ignore next */
+		const extraBytes = this.recordSize - (size % this.recordSize || this.recordSize);
+		this.written += extraBytes;
+
+		this.push(Buffer.concat([header, ...fileChunks, Buffer.alloc(extraBytes)], this.written - written));
+	}
+
+	public append(filepath: string, stats: Stats): void {
+		this.queue.push({
+			header: encodeHeader({
+				filename: relative(this.base, filepath),
+				mode: stats.mode,
+				uid: stats.uid,
+				gid: stats.gid,
+				size: stats.size,
+				mtime: Math.trunc(stats.mtime.valueOf() / 1000),
+				type: '0',
+				ustar: 'ustar ',
+				owner: '',
+				group: ''
+			}),
+			file: createReadStream(filepath),
+			size: stats.size
+		});
+	}
+
+}