From 470646b08a40d85331b88a2ea90b002d6238bf41 Mon Sep 17 00:00:00 2001 From: futpib Date: Sat, 16 Mar 2019 12:30:34 +0300 Subject: [PATCH 01/12] Add `globby.stream` --- index.js | 34 +++++++++++++++---- package.json | 2 ++ readme.md | 4 +++ stream-utils.js | 46 ++++++++++++++++++++++++++ test.js | 86 ++++++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 161 insertions(+), 11 deletions(-) create mode 100644 stream-utils.js diff --git a/index.js b/index.js index f8f3a57..029a19e 100644 --- a/index.js +++ b/index.js @@ -1,10 +1,12 @@ 'use strict'; const fs = require('fs'); const arrayUnion = require('array-union'); +const merge2 = require('merge2'); const glob = require('glob'); const fastGlob = require('fast-glob'); const dirGlob = require('dir-glob'); const gitignore = require('./gitignore'); +const {FilterStream, UniqueStream} = require('./stream-utils'); const DEFAULT_FILTER = () => false; @@ -81,6 +83,12 @@ const globDirs = (task, fn) => { const getPattern = (task, fn) => task.options.expandDirectories ? globDirs(task, fn) : [task.pattern]; +const getFilterSync = options => { + return options && options.gitignore ? + gitignore.sync({cwd: options.cwd, ignore: options.ignore}) : + DEFAULT_FILTER; +}; + const globToTask = task => glob => { const {options} = task; if (options.ignore && Array.isArray(options.ignore) && options.expandDirectories) { @@ -120,24 +128,36 @@ module.exports = async (patterns, options) => { module.exports.sync = (patterns, options) => { const globTasks = generateGlobTasks(patterns, options); - const getFilter = () => { - return options && options.gitignore ? - gitignore.sync({cwd: options.cwd, ignore: options.ignore}) : - DEFAULT_FILTER; - }; - const tasks = globTasks.reduce((tasks, task) => { const newTask = getPattern(task, dirGlob.sync).map(globToTask(task)); return tasks.concat(newTask); }, []); - const filter = getFilter(); + const filter = getFilterSync(options); + return tasks.reduce( (matches, task) => arrayUnion(matches, fastGlob.sync(task.pattern, task.options)), [] ).filter(path_ => !filter(path_)); }; +module.exports.stream = (patterns, options) => { + const globTasks = generateGlobTasks(patterns, options); + + const tasks = globTasks.reduce((tasks, task) => { + const newTask = getPattern(task, dirGlob.sync).map(globToTask(task)); + return tasks.concat(newTask); + }, []); + + const filter = getFilterSync(options); + const filterStream = new FilterStream(p => !filter(p)); + const uniqueStream = new UniqueStream(); + + return merge2(tasks.map(task => fastGlob.stream(task.pattern, task.options))) + .pipe(filterStream) + .pipe(uniqueStream); +}; + module.exports.generateGlobTasks = generateGlobTasks; module.exports.hasMagic = (patterns, options) => [] diff --git a/package.json b/package.json index cf84a8e..f0138de 100644 --- a/package.json +++ b/package.json @@ -61,10 +61,12 @@ "fast-glob": "^2.2.6", "glob": "^7.1.3", "ignore": "^5.1.1", + "merge2": "^1.2.3", "slash": "^3.0.0" }, "devDependencies": { "ava": "^2.1.0", + "get-stream": "^5.1.0", "glob-stream": "^6.1.0", "globby": "sindresorhus/globby#master", "matcha": "^0.7.0", diff --git a/readme.md b/readme.md index 46db037..b8b4ed2 100644 --- a/readme.md +++ b/readme.md @@ -93,6 +93,10 @@ Respect ignore patterns in `.gitignore` files that apply to the globbed files. Returns `string[]` of matching paths. +### globby.stream(patterns, options?) + +Returns a `ReadableStream` of matching paths. + ### globby.generateGlobTasks(patterns, options?) Returns an `object[]` in the format `{pattern: string, options: Object}`, which can be passed as arguments to [`fast-glob`](https://github.com/mrmlnc/fast-glob). This is useful for other globbing-related packages. diff --git a/stream-utils.js b/stream-utils.js new file mode 100644 index 0000000..98aedc8 --- /dev/null +++ b/stream-utils.js @@ -0,0 +1,46 @@ +'use strict'; +const {Transform} = require('stream'); + +class ObjectTransform extends Transform { + constructor() { + super({ + objectMode: true + }); + } +} + +class FilterStream extends ObjectTransform { + constructor(filter) { + super(); + this._filter = filter; + } + + _transform(data, encoding, callback) { + if (this._filter(data)) { + this.push(data); + } + + callback(); + } +} + +class UniqueStream extends ObjectTransform { + constructor() { + super(); + this._pushed = new Set(); + } + + _transform(data, encoding, callback) { + if (!this._pushed.has(data)) { + this.push(data); + this._pushed.add(data); + } + + callback(); + } +} + +module.exports = { + FilterStream, + UniqueStream +}; diff --git a/test.js b/test.js index 656bdfc..2403c3c 100644 --- a/test.js +++ b/test.js @@ -2,6 +2,7 @@ import fs from 'fs'; import util from 'util'; import path from 'path'; import test from 'ava'; +import getStream from 'get-stream'; import globby from '.'; const cwd = process.cwd(); @@ -72,6 +73,41 @@ test('return [] for all negative patterns - async', async t => { t.deepEqual(await globby(['!a.tmp', '!b.tmp']), []); }); +test('glob - stream', async t => { + t.deepEqual((await getStream.array(globby.stream('*.tmp'))).sort(), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']); +}); + +// Readable streams are readable since node version 10, but this test runs on 6 and 8 too. +// So we define the test only if async iteration is supported. +if (Symbol.asyncIterator) { + // For the reason behind `eslint-disable` below see https://github.com/avajs/eslint-plugin-ava/issues/216 + // eslint-disable-next-line ava/no-async-fn-without-await + test('glob - stream async iterator support', async t => { + const results = []; + for await (const path of globby.stream('*.tmp')) { + results.push(path); + } + + t.deepEqual(results, ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']); + }); +} + +test('glob - stream - multiple file paths', async t => { + t.deepEqual(await getStream.array(globby.stream(['a.tmp', 'b.tmp'])), ['a.tmp', 'b.tmp']); +}); + +test('glob with multiple patterns - stream', async t => { + t.deepEqual(await getStream.array(globby.stream(['a.tmp', '*.tmp', '!{c,d,e}.tmp'])), ['a.tmp', 'b.tmp']); +}); + +test('respect patterns order - stream', async t => { + t.deepEqual(await getStream.array(globby.stream(['!*.tmp', 'a.tmp'])), ['a.tmp']); +}); + +test('return [] for all negative patterns - stream', async t => { + t.deepEqual(await getStream.array(globby.stream(['!a.tmp', '!b.tmp'])), []); +}); + test('cwd option', t => { process.chdir(tmp); t.deepEqual(globby.sync('*.tmp', {cwd}), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']); @@ -89,6 +125,11 @@ test('don\'t mutate the options object - sync', t => { t.pass(); }); +test('don\'t mutate the options object - stream', async t => { + await getStream.array(globby.stream(['*.tmp', '!b.tmp'], Object.freeze({ignore: Object.freeze([])}))); + t.pass(); +}); + test('expose generateGlobTasks', t => { const tasks = globby.generateGlobTasks(['*.tmp', '!b.tmp'], {ignore: ['c.tmp']}); @@ -180,7 +221,7 @@ test.failing('relative paths and ignores option', t => { await t.throwsAsync(globby(value), message); }); - test(`throws for invalid patterns input: ${valueString}`, t => { + test(`throws for invalid patterns input: ${valueString} - sync`, t => { t.throws(() => { globby.sync(value); }, TypeError); @@ -190,6 +231,16 @@ test.failing('relative paths and ignores option', t => { }, message); }); + test(`throws for invalid patterns input: ${valueString} - stream`, t => { + t.throws(() => { + globby.stream(value); + }, TypeError); + + t.throws(() => { + globby.stream(value); + }, message); + }); + test(`generateGlobTasks throws for invalid patterns input: ${valueString}`, t => { t.throws(() => { globby.generateGlobTasks(value); @@ -201,7 +252,7 @@ test.failing('relative paths and ignores option', t => { }); }); -test('gitignore option defaults to false', async t => { +test('gitignore option defaults to false - async', async t => { const actual = await globby('*', {onlyFiles: false}); t.true(actual.includes('node_modules')); }); @@ -211,7 +262,12 @@ test('gitignore option defaults to false - sync', t => { t.true(actual.includes('node_modules')); }); -test('respects gitignore option true', async t => { +test('gitignore option defaults to false - stream', async t => { + const actual = await getStream.array(globby.stream('*', {onlyFiles: false})); + t.true(actual.indexOf('node_modules') > -1); +}); + +test('respects gitignore option true - async', async t => { const actual = await globby('*', {gitignore: true, onlyFiles: false}); t.false(actual.includes('node_modules')); }); @@ -221,7 +277,12 @@ test('respects gitignore option true - sync', t => { t.false(actual.includes('node_modules')); }); -test('respects gitignore option false', async t => { +test('respects gitignore option true - stream', async t => { + const actual = await getStream.array(globby.stream('*', {gitignore: true, onlyFiles: false})); + t.false(actual.indexOf('node_modules') > -1); +}); + +test('respects gitignore option false - async', async t => { const actual = await globby('*', {gitignore: false, onlyFiles: false}); t.true(actual.includes('node_modules')); }); @@ -237,6 +298,11 @@ test('gitignore option with stats option', async t => { t.false(actual.includes('node_modules')); }); +test('respects gitignore option false - stream', async t => { + const actual = await getStream.array(globby.stream('*', {gitignore: false, onlyFiles: false})); + t.true(actual.indexOf('node_modules') > -1); +}); + // https://github.com/sindresorhus/globby/issues/97 test.failing('`{extension: false}` and `expandDirectories.extensions` option', t => { t.deepEqual( @@ -284,3 +350,15 @@ test('throws when specifying a file as cwd - sync', t => { globby.sync('*', {cwd: isFile}); }, 'The `cwd` option must be a path to a directory'); }); + +test('throws when specifying a file as cwd - stream', t => { + const isFile = path.resolve('fixtures/gitignore/bar.js'); + + t.throws(() => { + globby.stream('.', {cwd: isFile}); + }, 'The `cwd` option must be a path to a directory'); + + t.throws(() => { + globby.stream('*', {cwd: isFile}); + }, 'The `cwd` option must be a path to a directory'); +}); From 30587a03dc01495454121e172c4a28db6c3fbe09 Mon Sep 17 00:00:00 2001 From: futpib Date: Fri, 14 Jun 2019 23:35:51 +0300 Subject: [PATCH 02/12] Update readme.md with a review suggestion Co-Authored-By: Sindre Sorhus --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index b8b4ed2..3bc529b 100644 --- a/readme.md +++ b/readme.md @@ -95,7 +95,7 @@ Returns `string[]` of matching paths. ### globby.stream(patterns, options?) -Returns a `ReadableStream` of matching paths. +Returns a [`stream.Readable`](https://nodejs.org/api/stream.html#stream_readable_streams) of matching paths. ### globby.generateGlobTasks(patterns, options?) From ba044fb5b73268bc48c0ae8a1ff1e3d974d51c4d Mon Sep 17 00:00:00 2001 From: futpib Date: Fri, 14 Jun 2019 23:41:17 +0300 Subject: [PATCH 03/12] Add an async iteration example to readme --- readme.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/readme.md b/readme.md index 3bc529b..7138a43 100644 --- a/readme.md +++ b/readme.md @@ -97,6 +97,14 @@ Returns `string[]` of matching paths. Returns a [`stream.Readable`](https://nodejs.org/api/stream.html#stream_readable_streams) of matching paths. +Since Node v10 [readable streams are iterable](https://nodejs.org/api/stream.html#stream_readable_symbol_asynciterator), so you can loop over glob matches in a [`for await...of` loop](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) like this: + +```js +for await (const path of globby.stream('*.tmp')) { + console.log({ path }); +} +``` + ### globby.generateGlobTasks(patterns, options?) Returns an `object[]` in the format `{pattern: string, options: Object}`, which can be passed as arguments to [`fast-glob`](https://github.com/mrmlnc/fast-glob). This is useful for other globbing-related packages. From e30d73d366636904418881571b972d8f16020834 Mon Sep 17 00:00:00 2001 From: futpib Date: Fri, 14 Jun 2019 23:41:29 +0300 Subject: [PATCH 04/12] Minor comment fix --- test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.js b/test.js index 2403c3c..11e745b 100644 --- a/test.js +++ b/test.js @@ -77,7 +77,7 @@ test('glob - stream', async t => { t.deepEqual((await getStream.array(globby.stream('*.tmp'))).sort(), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']); }); -// Readable streams are readable since node version 10, but this test runs on 6 and 8 too. +// Readable streams are iterable since node version 10, but this test runs on 6 and 8 too. // So we define the test only if async iteration is supported. if (Symbol.asyncIterator) { // For the reason behind `eslint-disable` below see https://github.com/avajs/eslint-plugin-ava/issues/216 From e3e933a756687d2726cea5521c29b1f214e88937 Mon Sep 17 00:00:00 2001 From: futpib Date: Fri, 14 Jun 2019 23:42:33 +0300 Subject: [PATCH 05/12] Add the missing file to package.json "files" field --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index f0138de..3c66658 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "files": [ "index.js", "gitignore.js", - "index.d.ts" + "index.d.ts", + "stream-utils.js" ], "keywords": [ "all", From 69940a0ab5dd1de0e28a689d879dfcee45944da4 Mon Sep 17 00:00:00 2001 From: futpib Date: Fri, 14 Jun 2019 23:59:49 +0300 Subject: [PATCH 06/12] Add `globby.stream` typescript definition --- index.d.ts | 10 ++++++++++ index.test-d.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/index.d.ts b/index.d.ts index b124362..13cfb5d 100644 --- a/index.d.ts +++ b/index.d.ts @@ -111,6 +111,16 @@ declare const globby: { options?: globby.GlobbyOptions ): string[]; + /** + @param patterns - See supported `minimatch` [patterns](https://github.com/isaacs/minimatch#usage). + @param options - See the [`fast-glob` options](https://github.com/mrmlnc/fast-glob#options-1) in addition to the ones in this package. + @returns The stream of matching paths. + */ + stream( + patterns: string | readonly string[], + options?: globby.GlobbyOptions + ): NodeJS.ReadableStream; + /** Note that you should avoid running the same tasks multiple times as they contain a file system cache. Instead, run this method each time to ensure file system changes are taken into consideration. diff --git a/index.test-d.ts b/index.test-d.ts index 4e7a3f0..e71dff2 100644 --- a/index.test-d.ts +++ b/index.test-d.ts @@ -4,6 +4,7 @@ import { GlobTask, FilterFunction, sync as globbySync, + stream as globbyStream, generateGlobTasks, hasMagic, gitignore @@ -45,6 +46,33 @@ expectType( expectType(globbySync('*.tmp', {gitignore: true})); expectType(globbySync('*.tmp', {ignore: ['**/b.tmp']})); +// Globby (stream) +expectType(globbyStream('*.tmp')); +expectType(globbyStream(['a.tmp', '*.tmp', '!{c,d,e}.tmp'])); + +expectType(globbyStream('*.tmp', {expandDirectories: false})); +expectType(globbyStream('*.tmp', {expandDirectories: ['a*', 'b*']})); +expectType( + globbyStream('*.tmp', { + expandDirectories: { + files: ['a', 'b'], + extensions: ['tmp'] + } + }) +); +expectType(globbyStream('*.tmp', {gitignore: true})); +expectType(globbyStream('*.tmp', {ignore: ['**/b.tmp']})); + +(async () => { + const streamResult = []; + for await (const path of globbyStream('*.tmp')) { + streamResult.push(path); + } + // `NodeJS.ReadableStream` is not generic, unfortunately, + // so it seems `(string | Buffer)[]` is the best we can get here + expectType<(string | Buffer)[]>(streamResult); +})(); + // GenerateGlobTasks expectType(generateGlobTasks('*.tmp')); expectType(generateGlobTasks(['a.tmp', '*.tmp', '!{c,d,e}.tmp'])); From c33b6ba05ca7c24cbb700cb659ae6f187c059cfe Mon Sep 17 00:00:00 2001 From: futpib Date: Sat, 15 Jun 2019 00:02:47 +0300 Subject: [PATCH 07/12] Use `includes` instead of `indexOf` --- test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test.js b/test.js index 11e745b..ce589eb 100644 --- a/test.js +++ b/test.js @@ -264,7 +264,7 @@ test('gitignore option defaults to false - sync', t => { test('gitignore option defaults to false - stream', async t => { const actual = await getStream.array(globby.stream('*', {onlyFiles: false})); - t.true(actual.indexOf('node_modules') > -1); + t.true(actual.includes('node_modules')); }); test('respects gitignore option true - async', async t => { @@ -279,7 +279,7 @@ test('respects gitignore option true - sync', t => { test('respects gitignore option true - stream', async t => { const actual = await getStream.array(globby.stream('*', {gitignore: true, onlyFiles: false})); - t.false(actual.indexOf('node_modules') > -1); + t.false(actual.includes('node_modules')); }); test('respects gitignore option false - async', async t => { @@ -300,7 +300,7 @@ test('gitignore option with stats option', async t => { test('respects gitignore option false - stream', async t => { const actual = await getStream.array(globby.stream('*', {gitignore: false, onlyFiles: false})); - t.true(actual.indexOf('node_modules') > -1); + t.true(actual.includes('node_modules')); }); // https://github.com/sindresorhus/globby/issues/97 From 6618d820fa6bd0580dae6d675984dbd4cfe5ab4a Mon Sep 17 00:00:00 2001 From: Sindre Sorhus Date: Sat, 15 Jun 2019 13:21:29 +0700 Subject: [PATCH 08/12] Update readme.md --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 7138a43..d84819e 100644 --- a/readme.md +++ b/readme.md @@ -97,11 +97,11 @@ Returns `string[]` of matching paths. Returns a [`stream.Readable`](https://nodejs.org/api/stream.html#stream_readable_streams) of matching paths. -Since Node v10 [readable streams are iterable](https://nodejs.org/api/stream.html#stream_readable_symbol_asynciterator), so you can loop over glob matches in a [`for await...of` loop](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) like this: +Since Node.js 10, [readable streams are iterable](https://nodejs.org/api/stream.html#stream_readable_symbol_asynciterator), so you can loop over glob matches in a [`for await...of` loop](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) like this: ```js for await (const path of globby.stream('*.tmp')) { - console.log({ path }); + console.log(path); } ``` From 34cfd208ad3b4a1900edd638c28ec3033bd9b899 Mon Sep 17 00:00:00 2001 From: Sindre Sorhus Date: Sat, 15 Jun 2019 13:22:58 +0700 Subject: [PATCH 09/12] Update test.js --- test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.js b/test.js index ce589eb..ffe776b 100644 --- a/test.js +++ b/test.js @@ -77,7 +77,7 @@ test('glob - stream', async t => { t.deepEqual((await getStream.array(globby.stream('*.tmp'))).sort(), ['a.tmp', 'b.tmp', 'c.tmp', 'd.tmp', 'e.tmp']); }); -// Readable streams are iterable since node version 10, but this test runs on 6 and 8 too. +// Readable streams are iterable since Node.js 10, but this test runs on 6 and 8 too. // So we define the test only if async iteration is supported. if (Symbol.asyncIterator) { // For the reason behind `eslint-disable` below see https://github.com/avajs/eslint-plugin-ava/issues/216 From af04fdc4a2c1e93ffb634af07ac96a5bf4f475d9 Mon Sep 17 00:00:00 2001 From: futpib Date: Sat, 15 Jun 2019 15:48:13 +0300 Subject: [PATCH 10/12] Add a jsdoc example of async iteration over a stream --- index.d.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/index.d.ts b/index.d.ts index 13cfb5d..429ad88 100644 --- a/index.d.ts +++ b/index.d.ts @@ -115,6 +115,17 @@ declare const globby: { @param patterns - See supported `minimatch` [patterns](https://github.com/isaacs/minimatch#usage). @param options - See the [`fast-glob` options](https://github.com/mrmlnc/fast-glob#options-1) in addition to the ones in this package. @returns The stream of matching paths. + + @example + ``` + import {stream} from 'globby'; + + (async () => { + for await (const path of stream('*.tmp')) { + console.log(path); + } + })(); + ``` */ stream( patterns: string | readonly string[], From 2d1b009487c2d6391076af50c5db0d8d63de2fa9 Mon Sep 17 00:00:00 2001 From: Sindre Sorhus Date: Sat, 15 Jun 2019 19:58:07 +0700 Subject: [PATCH 11/12] Update index.d.ts --- index.d.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.d.ts b/index.d.ts index 429ad88..49d669e 100644 --- a/index.d.ts +++ b/index.d.ts @@ -118,10 +118,10 @@ declare const globby: { @example ``` - import {stream} from 'globby'; + import globby = require('globby'); (async () => { - for await (const path of stream('*.tmp')) { + for await (const path of globby.stream('*.tmp')) { console.log(path); } })(); From 1debfbd0ccb6f4b1718632c904426344777e009e Mon Sep 17 00:00:00 2001 From: Sindre Sorhus Date: Sat, 15 Jun 2019 19:59:22 +0700 Subject: [PATCH 12/12] Update readme.md --- readme.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index d84819e..63859d0 100644 --- a/readme.md +++ b/readme.md @@ -67,6 +67,8 @@ Default: `true` If set to `true`, `globby` will automatically glob directories for you. If you define an `Array` it will only glob files that matches the patterns inside the `Array`. You can also define an `object` with `files` and `extensions` like below: ```js +const globby = require('globby'); + (async () => { const paths = await globby('images', { expandDirectories: { @@ -100,9 +102,13 @@ Returns a [`stream.Readable`](https://nodejs.org/api/stream.html#stream_readable Since Node.js 10, [readable streams are iterable](https://nodejs.org/api/stream.html#stream_readable_symbol_asynciterator), so you can loop over glob matches in a [`for await...of` loop](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) like this: ```js -for await (const path of globby.stream('*.tmp')) { - console.log(path); -} +const globby = require('globby'); + +(async () => { + for await (const path of globby.stream('*.tmp')) { + console.log(path); + } +})(); ``` ### globby.generateGlobTasks(patterns, options?)