Add globby.stream

index.js

@@ -1,10 +1,12 @@
 'use strict';
 const fs = require('fs');
 const arrayUnion = require('array-union');
+const merge2 = require('merge2');
 const glob = require('glob');
 const fastGlob = require('fast-glob');
 const dirGlob = require('dir-glob');
 const gitignore = require('./gitignore');
+const {FilterStream, UniqueStream} = require('./stream-utils');
 
 const DEFAULT_FILTER = () => false;
 
@@ -71,6 +73,12 @@ const globDirs = (task, fn) => {
 
 const getPattern = (task, fn) => task.options.expandDirectories ? globDirs(task, fn) : [task.pattern];
 
+const getFilterSync = options => {
+	return options && options.gitignore ?
+		gitignore.sync({cwd: options.cwd, ignore: options.ignore}) :
+		DEFAULT_FILTER;
+};
+
 const globToTask = task => glob => {
 	const {options} = task;
 	if (options.ignore && Array.isArray(options.ignore) && options.expandDirectories) {
@@ -120,24 +128,36 @@ module.exports.default = globby;
 module.exports.sync = (patterns, options) => {
 	const globTasks = generateGlobTasks(patterns, options);
 
-	const getFilter = () => {
-		return options && options.gitignore ?
-			gitignore.sync({cwd: options.cwd, ignore: options.ignore}) :
-			DEFAULT_FILTER;
-	};
-
 	const tasks = globTasks.reduce((tasks, task) => {
 		const newTask = getPattern(task, dirGlob.sync).map(globToTask(task));
 		return tasks.concat(newTask);
 	}, []);
 
-	const filter = getFilter();
+	const filter = getFilterSync(options);
+
 	return tasks.reduce(
 		(matches, task) => arrayUnion(matches, fastGlob.sync(task.pattern, task.options)),
 		[]
 	).filter(p => !filter(p));
 };
 
+module.exports.stream = (patterns, options) => {
+	const globTasks = generateGlobTasks(patterns, options);
+
+	const tasks = globTasks.reduce((tasks, task) => {
+		const newTask = getPattern(task, dirGlob.sync).map(globToTask(task));
+		return tasks.concat(newTask);
+	}, []);
+
+	const filter = getFilterSync(options);
+	const filterStream = new FilterStream(p => !filter(p));
+	const uniqueStream = new UniqueStream();
+
+	return merge2(tasks.map(task => fastGlob.stream(task.pattern, task.options)))
+		.pipe(filterStream)
+		.pipe(uniqueStream);
+};
+
 module.exports.generateGlobTasks = generateGlobTasks;
 
 module.exports.hasMagic = (patterns, options) => []

package.json

@@ -62,11 +62,13 @@
 		"fast-glob": "^2.2.6",
 		"glob": "^7.1.3",
 		"ignore": "^4.0.3",
+		"merge2": "^1.2.3",
 		"pify": "^4.0.1",
 		"slash": "^2.0.0"
 	},
 	"devDependencies": {
 		"ava": "^1.2.1",
+		"get-stream": "4.1.0",
 		"glob-stream": "^6.1.0",
 		"globby": "sindresorhus/globby#master",
 		"matcha": "^0.7.0",

readme.md

@@ -93,6 +93,10 @@ Respect ignore patterns in `.gitignore` files that apply to the globbed files.
 
 Returns an `Array` of matching paths.
 
+### globby.stream(patterns, [options])
+
+Returns a `ReadableStream` of matching paths.
+
 ### globby.generateGlobTasks(patterns, [options])
 
 Returns an `Array<Object>` in the format `{pattern: string, options: Object}`, which can be passed as arguments to [`fast-glob`](https://github.com/mrmlnc/fast-glob). This is useful for other globbing-related packages.

stream-utils.js

@@ -0,0 +1,46 @@
+'use strict';
+const {Transform} = require('stream');
+
+class ObjectTransform extends Transform {
+	constructor() {
+		super({
+			objectMode: true
+		});
+	}
+}
+
+class FilterStream extends ObjectTransform {
+	constructor(filter) {
+		super();
+		this._filter = filter;
+	}
+
+	_transform(data, encoding, callback) {
+		if (this._filter(data)) {
+			this.push(data);
+		}
+
+		callback();
+	}
+}
+
+class UniqueStream extends ObjectTransform {
+	constructor() {
+		super();
+		this._pushed = new Set();
+	}
+
+	_transform(data, encoding, callback) {
+		if (!this._pushed.has(data)) {
+			this.push(data);
+			this._pushed.add(data);
+		}
+
+		callback();
+	}
+}
+
+module.exports = {
+	FilterStream,
+	UniqueStream
+};

test.js

@@ -2,6 +2,7 @@ import fs from 'fs';
 import util from 'util';
 import path from 'path';
 import test from 'ava';
+import getStream from 'get-stream';
 import globby from '.';
 
 const cwd = process.cwd();
@@ -72,6 +73,41 @@ test('return [] for all negative patterns - async', async t => {
 	t.deepEqual(await globby(['!a.tmp', '!b.tmp']), []);
 });
 
+test('glob - stream', async t => {
+	t.deepEqual((await getStream.array(globby.stream('*.tmp'))).sort(), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']);
+});
+
+// Readable streams are readable since node version 10, but this test runs on 6 and 8 too.
+// So we define the test only if async iteration is supported.
+if (Symbol.asyncIterator) {
+	// For the reason behind `eslint-disable` below see https://github.com/avajs/eslint-plugin-ava/issues/216
+	// eslint-disable-next-line ava/no-async-fn-without-await
+	test('glob - stream async iterator support', async t => {
+		const results = [];
+		for await (const path of globby.stream('*.tmp')) {
+			results.push(path);
+		}
+
+		t.deepEqual(results, ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']);
+	});
+}
+
+test('glob - stream - multiple file paths', async t => {
+	t.deepEqual(await getStream.array(globby.stream(['a.tmp', 'b.tmp'])), ['a.tmp', 'b.tmp']);
+});
+
+test('glob with multiple patterns - stream', async t => {
+	t.deepEqual(await getStream.array(globby.stream(['a.tmp', '*.tmp', '!{c,d,e}.tmp'])), ['a.tmp', 'b.tmp']);
+});
+
+test('respect patterns order - stream', async t => {
+	t.deepEqual(await getStream.array(globby.stream(['!*.tmp', 'a.tmp'])), ['a.tmp']);
+});
+
+test('return [] for all negative patterns - stream', async t => {
+	t.deepEqual(await getStream.array(globby.stream(['!a.tmp', '!b.tmp'])), []);
+});
+
 test('cwd option', t => {
 	process.chdir(tmp);
 	t.deepEqual(globby.sync('*.tmp', {cwd}), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']);
@@ -89,6 +125,11 @@ test('don\'t mutate the options object - sync', t => {
 	t.pass();
 });
 
+test('don\'t mutate the options object - stream', async t => {
+	await getStream.array(globby.stream(['*.tmp', '!b.tmp'], Object.freeze({ignore: Object.freeze([])})));
+	t.pass();
+});
+
 test('expose generateGlobTasks', t => {
 	const tasks = globby.generateGlobTasks(['*.tmp', '!b.tmp'], {ignore: ['c.tmp']});
 
@@ -180,18 +221,23 @@ test.failing('relative paths and ignores option', t => {
 		await t.throwsAsync(globby(v), msg);
 	});
 
-	test(`throws for invalid patterns input: ${valstring}`, t => {
+	test(`throws for invalid patterns input: ${valstring} - sync`, t => {
 		t.throws(() => globby.sync(v), TypeError);
 		t.throws(() => globby.sync(v), msg);
 	});
 
+	test(`throws for invalid patterns input: ${valstring} - stream`, t => {
+		t.throws(() => globby.stream(v), TypeError);
+		t.throws(() => globby.stream(v), msg);
+	});
+
 	test(`generateGlobTasks throws for invalid patterns input: ${valstring}`, t => {
 		t.throws(() => globby.generateGlobTasks(v), TypeError);
 		t.throws(() => globby.generateGlobTasks(v), msg);
 	});
 });
 
-test('gitignore option defaults to false', async t => {
+test('gitignore option defaults to false - async', async t => {
 	const actual = await globby('*', {onlyFiles: false});
 	t.true(actual.indexOf('node_modules') > -1);
 });
@@ -201,7 +247,12 @@ test('gitignore option defaults to false - sync', t => {
 	t.true(actual.indexOf('node_modules') > -1);
 });
 
-test('respects gitignore option true', async t => {
+test('gitignore option defaults to false - stream', async t => {
+	const actual = await getStream.array(globby.stream('*', {onlyFiles: false}));
+	t.true(actual.indexOf('node_modules') > -1);
+});
+
+test('respects gitignore option true - async', async t => {
 	const actual = await globby('*', {gitignore: true, onlyFiles: false});
 	t.false(actual.indexOf('node_modules') > -1);
 });
@@ -211,7 +262,12 @@ test('respects gitignore option true - sync', t => {
 	t.false(actual.indexOf('node_modules') > -1);
 });
 
-test('respects gitignore option false', async t => {
+test('respects gitignore option true - stream', async t => {
+	const actual = await getStream.array(globby.stream('*', {gitignore: true, onlyFiles: false}));
+	t.false(actual.indexOf('node_modules') > -1);
+});
+
+test('respects gitignore option false - async', async t => {
 	const actual = await globby('*', {gitignore: false, onlyFiles: false});
 	t.true(actual.indexOf('node_modules') > -1);
 });
@@ -221,6 +277,11 @@ test('respects gitignore option false - sync', t => {
 	t.true(actual.indexOf('node_modules') > -1);
 });
 
+test('respects gitignore option false - stream', async t => {
+	const actual = await getStream.array(globby.stream('*', {gitignore: false, onlyFiles: false}));
+	t.true(actual.indexOf('node_modules') > -1);
+});
+
 // https://github.com/sindresorhus/globby/issues/97
 test.failing('`{extension: false}` and `expandDirectories.extensions` option', t => {
 	t.deepEqual(
@@ -268,3 +329,15 @@ test('throws when specifying a file as cwd - sync', t => {
 		globby.sync('*', {cwd: isFile});
 	}, 'The `cwd` option must be a path to a directory');
 });
+
+test('throws when specifying a file as cwd - stream', t => {
+	const isFile = path.resolve('fixtures/gitignore/bar.js');
+
+	t.throws(() => {
+		globby.stream('.', {cwd: isFile});
+	}, 'The `cwd` option must be a path to a directory');
+
+	t.throws(() => {
+		globby.stream('*', {cwd: isFile});
+	}, 'The `cwd` option must be a path to a directory');
+});