diff --git a/index.js b/index.js index 08c3150..e22beb4 100644 --- a/index.js +++ b/index.js @@ -1,10 +1,12 @@ 'use strict'; const fs = require('fs'); const arrayUnion = require('array-union'); +const merge2 = require('merge2'); const glob = require('glob'); const fastGlob = require('fast-glob'); const dirGlob = require('dir-glob'); const gitignore = require('./gitignore'); +const {FilterStream, UniqueStream} = require('./stream-utils'); const DEFAULT_FILTER = () => false; @@ -71,6 +73,12 @@ const globDirs = (task, fn) => { const getPattern = (task, fn) => task.options.expandDirectories ? globDirs(task, fn) : [task.pattern]; +const getFilterSync = options => { + return options && options.gitignore ? + gitignore.sync({cwd: options.cwd, ignore: options.ignore}) : + DEFAULT_FILTER; +}; + const globToTask = task => glob => { const {options} = task; if (options.ignore && Array.isArray(options.ignore) && options.expandDirectories) { @@ -120,24 +128,36 @@ module.exports.default = globby; module.exports.sync = (patterns, options) => { const globTasks = generateGlobTasks(patterns, options); - const getFilter = () => { - return options && options.gitignore ? - gitignore.sync({cwd: options.cwd, ignore: options.ignore}) : - DEFAULT_FILTER; - }; - const tasks = globTasks.reduce((tasks, task) => { const newTask = getPattern(task, dirGlob.sync).map(globToTask(task)); return tasks.concat(newTask); }, []); - const filter = getFilter(); + const filter = getFilterSync(options); + return tasks.reduce( (matches, task) => arrayUnion(matches, fastGlob.sync(task.pattern, task.options)), [] ).filter(p => !filter(p)); }; +module.exports.stream = (patterns, options) => { + const globTasks = generateGlobTasks(patterns, options); + + const tasks = globTasks.reduce((tasks, task) => { + const newTask = getPattern(task, dirGlob.sync).map(globToTask(task)); + return tasks.concat(newTask); + }, []); + + const filter = getFilterSync(options); + const filterStream = new FilterStream(p => !filter(p)); + const uniqueStream = new UniqueStream(); + + return merge2(tasks.map(task => fastGlob.stream(task.pattern, task.options))) + .pipe(filterStream) + .pipe(uniqueStream); +}; + module.exports.generateGlobTasks = generateGlobTasks; module.exports.hasMagic = (patterns, options) => [] diff --git a/package.json b/package.json index 245435e..01f0f26 100644 --- a/package.json +++ b/package.json @@ -62,11 +62,13 @@ "fast-glob": "^2.2.6", "glob": "^7.1.3", "ignore": "^4.0.3", + "merge2": "^1.2.3", "pify": "^4.0.1", "slash": "^2.0.0" }, "devDependencies": { "ava": "^1.2.1", + "get-stream": "4.1.0", "glob-stream": "^6.1.0", "globby": "sindresorhus/globby#master", "matcha": "^0.7.0", diff --git a/readme.md b/readme.md index 70c2b84..ba3b6f5 100644 --- a/readme.md +++ b/readme.md @@ -93,6 +93,10 @@ Respect ignore patterns in `.gitignore` files that apply to the globbed files. Returns an `Array` of matching paths. +### globby.stream(patterns, [options]) + +Returns a `ReadableStream` of matching paths. + ### globby.generateGlobTasks(patterns, [options]) Returns an `Array` in the format `{pattern: string, options: Object}`, which can be passed as arguments to [`fast-glob`](https://github.com/mrmlnc/fast-glob). This is useful for other globbing-related packages. diff --git a/stream-utils.js b/stream-utils.js new file mode 100644 index 0000000..98aedc8 --- /dev/null +++ b/stream-utils.js @@ -0,0 +1,46 @@ +'use strict'; +const {Transform} = require('stream'); + +class ObjectTransform extends Transform { + constructor() { + super({ + objectMode: true + }); + } +} + +class FilterStream extends ObjectTransform { + constructor(filter) { + super(); + this._filter = filter; + } + + _transform(data, encoding, callback) { + if (this._filter(data)) { + this.push(data); + } + + callback(); + } +} + +class UniqueStream extends ObjectTransform { + constructor() { + super(); + this._pushed = new Set(); + } + + _transform(data, encoding, callback) { + if (!this._pushed.has(data)) { + this.push(data); + this._pushed.add(data); + } + + callback(); + } +} + +module.exports = { + FilterStream, + UniqueStream +}; diff --git a/test.js b/test.js index 53acc35..4ac149c 100644 --- a/test.js +++ b/test.js @@ -2,6 +2,7 @@ import fs from 'fs'; import util from 'util'; import path from 'path'; import test from 'ava'; +import getStream from 'get-stream'; import globby from '.'; const cwd = process.cwd(); @@ -72,6 +73,41 @@ test('return [] for all negative patterns - async', async t => { t.deepEqual(await globby(['!a.tmp', '!b.tmp']), []); }); +test('glob - stream', async t => { + t.deepEqual((await getStream.array(globby.stream('*.tmp'))).sort(), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']); +}); + +// Readable streams are readable since node version 10, but this test runs on 6 and 8 too. +// So we define the test only if async iteration is supported. +if (Symbol.asyncIterator) { + // For the reason behind `eslint-disable` below see https://github.com/avajs/eslint-plugin-ava/issues/216 + // eslint-disable-next-line ava/no-async-fn-without-await + test('glob - stream async iterator support', async t => { + const results = []; + for await (const path of globby.stream('*.tmp')) { + results.push(path); + } + + t.deepEqual(results, ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']); + }); +} + +test('glob - stream - multiple file paths', async t => { + t.deepEqual(await getStream.array(globby.stream(['a.tmp', 'b.tmp'])), ['a.tmp', 'b.tmp']); +}); + +test('glob with multiple patterns - stream', async t => { + t.deepEqual(await getStream.array(globby.stream(['a.tmp', '*.tmp', '!{c,d,e}.tmp'])), ['a.tmp', 'b.tmp']); +}); + +test('respect patterns order - stream', async t => { + t.deepEqual(await getStream.array(globby.stream(['!*.tmp', 'a.tmp'])), ['a.tmp']); +}); + +test('return [] for all negative patterns - stream', async t => { + t.deepEqual(await getStream.array(globby.stream(['!a.tmp', '!b.tmp'])), []); +}); + test('cwd option', t => { process.chdir(tmp); t.deepEqual(globby.sync('*.tmp', {cwd}), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']); @@ -89,6 +125,11 @@ test('don\'t mutate the options object - sync', t => { t.pass(); }); +test('don\'t mutate the options object - stream', async t => { + await getStream.array(globby.stream(['*.tmp', '!b.tmp'], Object.freeze({ignore: Object.freeze([])}))); + t.pass(); +}); + test('expose generateGlobTasks', t => { const tasks = globby.generateGlobTasks(['*.tmp', '!b.tmp'], {ignore: ['c.tmp']}); @@ -180,18 +221,23 @@ test.failing('relative paths and ignores option', t => { await t.throwsAsync(globby(v), msg); }); - test(`throws for invalid patterns input: ${valstring}`, t => { + test(`throws for invalid patterns input: ${valstring} - sync`, t => { t.throws(() => globby.sync(v), TypeError); t.throws(() => globby.sync(v), msg); }); + test(`throws for invalid patterns input: ${valstring} - stream`, t => { + t.throws(() => globby.stream(v), TypeError); + t.throws(() => globby.stream(v), msg); + }); + test(`generateGlobTasks throws for invalid patterns input: ${valstring}`, t => { t.throws(() => globby.generateGlobTasks(v), TypeError); t.throws(() => globby.generateGlobTasks(v), msg); }); }); -test('gitignore option defaults to false', async t => { +test('gitignore option defaults to false - async', async t => { const actual = await globby('*', {onlyFiles: false}); t.true(actual.indexOf('node_modules') > -1); }); @@ -201,7 +247,12 @@ test('gitignore option defaults to false - sync', t => { t.true(actual.indexOf('node_modules') > -1); }); -test('respects gitignore option true', async t => { +test('gitignore option defaults to false - stream', async t => { + const actual = await getStream.array(globby.stream('*', {onlyFiles: false})); + t.true(actual.indexOf('node_modules') > -1); +}); + +test('respects gitignore option true - async', async t => { const actual = await globby('*', {gitignore: true, onlyFiles: false}); t.false(actual.indexOf('node_modules') > -1); }); @@ -211,7 +262,12 @@ test('respects gitignore option true - sync', t => { t.false(actual.indexOf('node_modules') > -1); }); -test('respects gitignore option false', async t => { +test('respects gitignore option true - stream', async t => { + const actual = await getStream.array(globby.stream('*', {gitignore: true, onlyFiles: false})); + t.false(actual.indexOf('node_modules') > -1); +}); + +test('respects gitignore option false - async', async t => { const actual = await globby('*', {gitignore: false, onlyFiles: false}); t.true(actual.indexOf('node_modules') > -1); }); @@ -221,6 +277,11 @@ test('respects gitignore option false - sync', t => { t.true(actual.indexOf('node_modules') > -1); }); +test('respects gitignore option false - stream', async t => { + const actual = await getStream.array(globby.stream('*', {gitignore: false, onlyFiles: false})); + t.true(actual.indexOf('node_modules') > -1); +}); + // https://github.com/sindresorhus/globby/issues/97 test.failing('`{extension: false}` and `expandDirectories.extensions` option', t => { t.deepEqual( @@ -268,3 +329,15 @@ test('throws when specifying a file as cwd - sync', t => { globby.sync('*', {cwd: isFile}); }, 'The `cwd` option must be a path to a directory'); }); + +test('throws when specifying a file as cwd - stream', t => { + const isFile = path.resolve('fixtures/gitignore/bar.js'); + + t.throws(() => { + globby.stream('.', {cwd: isFile}); + }, 'The `cwd` option must be a path to a directory'); + + t.throws(() => { + globby.stream('*', {cwd: isFile}); + }, 'The `cwd` option must be a path to a directory'); +});