/
package-cache.js
678 lines (607 loc) · 20.7 KB
/
package-cache.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
'use strict';
const fs = require('fs-extra');
const path = require('path');
const quickTemp = require('quick-temp');
const Configstore = require('configstore');
const CommandGenerator = require('./command-generator');
const stableStringify = require('safe-stable-stringify');
const symlinkOrCopySync = require('symlink-or-copy').sync;
let originalWorkingDirectory = process.cwd();
// Module scoped variable to store whether a particular cache has been
// attempted to be upgraded.
let upgraded = {};
/*
List of keys which could possibly result in the package manager installing
something. This is the "err on the side of caution" approach. It actually
doesn't matter if something is or isn't automatically installed in any of the
cases where we use this.
*/
let DEPENDENCY_KEYS = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies'];
/**
* The `bower` command helper.
*
* @private
* @method bower
* @param {String} subcommand The subcommand to be passed into bower.
* @param {String} [...arguments] Arguments to be passed into the bower subcommand.
* @param {Object} [options={}] The options passed into child_process.spawnSync.
* (https://nodejs.org/api/child_process.html#child_process_child_process_spawnsync_command_args_options)
*/
let bower = new CommandGenerator('bower');
/**
* The `npm` command helper.
*
* @private
* @method npm
* @param {String} subcommand The subcommand to be passed into npm.
* @param {String} [...arguments] Arguments to be passed into the npm subcommand.
* @param {Object} [options={}] The options passed into child_process.spawnSync.
* (https://nodejs.org/api/child_process.html#child_process_child_process_spawnsync_command_args_options)
*/
let npm = new CommandGenerator('npm');
/**
* The `yarn` command helper.
*
* @private
* @method yarn
* @param {String} subcommand The subcommand to be passed into yarn.
* @param {String} [...arguments] Arguments to be passed into the yarn subcommand.
* @param {Object} [options={}] The options passed into child_process.spawnSync.
* (https://nodejs.org/api/child_process.html#child_process_child_process_spawnsync_command_args_options)
*/
let yarn = new CommandGenerator('yarn');
// This lookup exists to make it possible to look the commands up based upon context.
let originals;
let commands = {
bower,
npm,
yarn,
};
// The definition list of translation terms.
let lookups = {
manifest: {
bower: 'bower.json',
npm: 'package.json',
yarn: 'package.json',
},
path: {
bower: 'bower_components',
npm: 'node_modules',
yarn: 'node_modules',
},
upgrade: {
bower: 'update',
npm: 'install',
yarn: 'upgrade',
},
};
/**
* The `translate` command is used to turn a consistent argument into
* appropriate values based upon the context in which it is used. It's
* a convenience helper to avoid littering lookups throughout the code.
*
* @private
* @method translate
* @param {String} type Either 'bower', 'npm', or 'yarn'.
* @param {String} lookup Either 'manifest', 'path', or 'upgrade'.
*/
function translate(type, lookup) {
return lookups[lookup][type];
}
/**
* The PackageCache wraps all package management functions. It also
* handles initial global state setup.
*
* Usage:
*
* ```
* let cache = new PackageCache();
* let dir = cache.create('your-cache', 'yarn', '{
* "dependencies": {
* "lodash": "*",
* "ember-cli": "*"
* }
* }');
* // => process.cwd()/tmp/your-cache-A3B4C6D
* ```
*
* This will generate a persistent cache which contains the results
* of a clean installation of the `dependencies` as you specified in
* the manifest argument. It will save the results of these in a
* temporary folder, returned as `dir`. On a second invocation
* (in the same process, or in a subsequent run) PackageCache will first
* compare the manifest to the previously installed one, using the manifest
* as the cache key, and make a decision as to the fastest way to get
* the cache up-to-date. PackageCache guarantees that your cache will
* always be up-to-date.
*
* If done in the same process, this simply returns the existing cache
* directory with no change, making the following invocation simply a
* cache validation check:
*
* ```
* let dir2 = cache.create('your-cache', 'yarn', '{
* "dependencies": {
* "lodash": "*",
* "ember-cli": "*"
* }
* }');
* // => process.cwd()/tmp/your-cache-A3B4C6D
* ```
*
* If you wish to modify a cache you can do so using the `update` API:
*
* ```
* let dir3 = cache.update('your-cache', 'yarn', '{
* "dependencies": {
* "": "*",
* "lodash": "*",
* "ember-cli": "*"
* }
* }');
* // => process.cwd()/tmp/your-cache-A3B4C6D
* ```
*
* Underneath the hood `create` and `update` are identical–which
* makes clear the simplicity of this tool. It will always do the
* right thing. You can think of the outcome of any `create` or
* `update` call as identical to `rm -rf node_modules && npm install`
* except as performant as possible.
*
* If you need to make modifications to a cache but wish to retain
* the original you can invoke the `clone` command:
*
* ```
* let newDir = cache.clone('your-cache', 'modified-cache');
* let manifest = fs.readJsonSync(path.join(newDir, 'package.json'));
* manifest.dependencies['express'] = '*';
* cache.update('modified-cache', 'yarn', JSON.stringify(manifest));
* // => process.cwd()/tmp/modified-cache-F8D5C8B
* ```
*
* This mental model makes it easy to prevent coding mistakes, state
* leakage across multiple test runs by making multiple caches cheap,
* and has tremendous performance benefits.
*
* You can even programatically update a cache:
*
* ```
* let CommandGenerator = require('./command-generator');
* let yarn = new CommandGenerator('yarn');
*
* let dir = cache.create('your-cache', 'yarn', '{ ... }');
*
* yarn.invoke('add', 'some-addon', { cwd: dir });
* ```
*
* The programmatic approach enables the entire set of usecases that
* the underlying package manager supports while continuing to wrap it
* in a persistent cache. You should not directly modify any files in the
* cache other than the manifest unless you really know what you're doing as
* that can put the cache into a possibly invalid state.
*
* PackageCache also supports linking. If you pass an array of module names to
* in the fourth position it will ensure that those are linked, and remain
* linked for the lifetime of the cache. When you link a package it uses the
* current global link provided by the underlying package manager and invokes
* their built-in `link` command.
*
* ```
* let dir = cache.create('your-cache', 'yarn', '{ ... }', ['ember-cli']);
* // => `yarn link ember-cli` happens along the way.
* ```
*
* Sometimes this global linking behavior is not what you want. Instead you wish
* to link in some other working directory. PackageCache makes a best effort
* attempt at supporting this workflow by allowing you to specify an object in
* the `links` argument array passed to `create`.
*
* let dir = cache.create('your-cache', 'yarn', '{ ... }', [
* { name: 'ember-cli', path: '/the/absolute/path/to/the/package' },
* 'other-package'
* ]);
*
* This creates a symlink at the named package path to the specified directory.
* As package managers do different things for their own linking behavior this
* is "best effort" only. Be sure to review upstream behavior to identify if you
* rely on those features for your code to function:
*
* - [bower](https://github.com/bower/bower/blob/master/lib/commands/link.js)
* - [npm](https://github.com/npm/npm/blob/latest/lib/link.js)
* - [yarn](https://github.com/yarnpkg/yarn/blob/master/src/cli/commands/link.js)
*
* As the only caveat, PackageCache _is_ persistent. The consuming
* code is responsible for ensuring that the cache size does not
* grow unbounded.
*
* @private
* @class PackageCache
* @constructor
* @param {String} rootPath Root of the directory for `PackageCache`.
*/
module.exports = class PackageCache {
constructor(rootPath) {
this.rootPath = rootPath || originalWorkingDirectory;
let configPath = path.join(this.rootPath, 'tmp', 'package-cache.json');
this._conf = new Configstore(
'package-cache',
{},
{
configPath,
}
);
this._cleanDirs();
}
get dirs() {
return this._conf.all;
}
/**
* The `__setupForTesting` modifies things in module scope.
*
* @private
* @method __setupForTesting
*/
__setupForTesting(stubs) {
originals = commands;
commands = stubs.commands;
}
/**
* The `__resetForTesting` puts things back in module scope.
*
* @private
* @method __resetForTesting
*/
__resetForTesting() {
commands = originals;
}
/**
* The `_cleanDirs` method deals with sync issues between the
* Configstore and what exists on disk. Non-existent directories
* are removed from `this.dirs`.
*
* @private
* @method _cleanDirs
*/
_cleanDirs() {
let labels = Object.keys(this.dirs);
let label, directory;
for (let i = 0; i < labels.length; i++) {
label = labels[i];
directory = this.dirs[label];
if (!fs.existsSync(directory)) {
this._conf.delete(label);
}
}
}
/**
* The `_readManifest` method reads the on-disk manifest for the current
* cache and returns its value.
*
* @private
* @method _readManifest
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
* @return {String} The manifest file contents on disk.
*/
_readManifest(label, type) {
let readManifestDir = this.dirs[label];
if (!readManifestDir) {
return null;
}
let inputPath = path.join(readManifestDir, translate(type, 'manifest'));
let result = null;
try {
result = fs.readFileSync(inputPath, 'utf8');
} catch (error) {
// Swallow non-exceptional errors.
if (error.code !== 'ENOENT') {
throw error;
}
}
return result;
}
/**
* The `_writeManifest` method generates the on-disk folder for the package cache
* and saves the manifest into it. If it is a yarn package cache it will remove
* the existing lock file.
*
* @private
* @method _writeManifest
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
* @param {String} manifest The contents of the manifest file to write to disk.
*/
_writeManifest(label, type, manifest) {
process.chdir(this.rootPath);
let outputDir = quickTemp.makeOrReuse(this.dirs, label);
process.chdir(originalWorkingDirectory);
this._conf.set(label, outputDir);
let outputFile = path.join(outputDir, translate(type, 'manifest'));
fs.outputFileSync(outputFile, manifest);
// Remove any existing yarn.lock file so that it doesn't try to incorrectly use it as a base.
if (type === 'yarn') {
try {
fs.unlinkSync(path.join(outputDir, 'yarn.lock'));
} catch (error) {
// Catch unexceptional error but rethrow if something is truly wrong.
if (error.code !== 'ENOENT') {
throw error;
}
}
}
}
/**
* The `_removeLinks` method removes from the dependencies of the manifest the
* assets which will be linked in so that we don't duplicate install. It saves
* off the values in the internal `PackageCache` metadata for restoration after
* linking as those values may be necessary.
*
* It is also responsible for removing these links prior to making any changes
* to the specified cache.
*
* @private
* @method _removeLinks
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
*/
_removeLinks(label, type) {
let cachedManifest = this._readManifest(label, type);
if (!cachedManifest) {
return;
}
let jsonManifest = JSON.parse(cachedManifest);
let links = jsonManifest._packageCache.links;
// Blindly remove existing links whether or not they appear in the manifest.
let link, linkPath;
for (let i = 0; i < links.length; i++) {
link = links[i];
if (typeof link === 'string') {
commands[type].invoke('unlink', link, { cwd: this.dirs[label] });
} else {
linkPath = path.join(this.dirs[label], translate(type, 'path'), link.name);
try {
fs.removeSync(linkPath);
} catch (error) {
// Catch unexceptional error but rethrow if something is truly wrong.
if (error.code !== 'ENOENT') {
throw error;
}
}
}
}
// Remove things from the manifest which we know we'll link back in.
let originals = {};
let key, linkName;
for (let i = 0; i < DEPENDENCY_KEYS.length; i++) {
key = DEPENDENCY_KEYS[i];
if (jsonManifest[key]) {
// Get a clone of the original object.
originals[key] = JSON.parse(JSON.stringify(jsonManifest[key]));
}
for (let j = 0; j < links.length; j++) {
link = links[j];
// Support object-style invocation for "manual" linking.
if (typeof link === 'string') {
linkName = link;
} else {
linkName = link.name;
}
if (jsonManifest[key] && jsonManifest[key][linkName]) {
delete jsonManifest[key][linkName];
}
}
}
jsonManifest._packageCache.originals = originals;
let manifest = JSON.stringify(jsonManifest);
this._writeManifest(label, type, manifest);
}
/**
* The `_restoreLinks` method restores the dependencies from the internal
* `PackageCache` metadata so that the manifest matches its original state after
* performing the links.
*
* It is also responsible for restoring these links into the `PackageCache`.
*
* @private
* @method _restoreLinks
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
*/
_restoreLinks(label, type) {
let cachedManifest = this._readManifest(label, type);
if (!cachedManifest) {
return;
}
let jsonManifest = JSON.parse(cachedManifest);
let links = jsonManifest._packageCache.links;
// Blindly restore links.
let link, linkPath;
for (let i = 0; i < links.length; i++) {
link = links[i];
if (typeof link === 'string') {
commands[type].invoke('link', link, { cwd: this.dirs[label] });
} else {
linkPath = path.join(this.dirs[label], translate(type, 'path'), link.name);
fs.mkdirsSync(path.dirname(linkPath)); // Just in case the path doesn't exist.
symlinkOrCopySync(link.path, linkPath);
}
}
// Restore to original state.
let keys = Object.keys(jsonManifest._packageCache.originals);
let key;
for (let i = 0; i < keys.length; i++) {
key = keys[i];
jsonManifest[key] = jsonManifest._packageCache.originals[key];
}
// Get rid of the originals.
delete jsonManifest._packageCache.originals;
// Serialize back to disk.
let manifest = JSON.stringify(jsonManifest);
this._writeManifest(label, type, manifest);
}
/**
* The `_checkManifest` method compares the desired manifest to that which
* exists in the cache.
*
* @private
* @method _checkManifest
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
* @param {String} manifest The contents of the manifest file to compare to cache.
* @return {Boolean} `true` if identical.
*/
_checkManifest(label, type, manifest) {
let cachedManifest = this._readManifest(label, type);
if (cachedManifest === null) {
return false;
}
let parsedCached = JSON.parse(cachedManifest);
let parsedNew = JSON.parse(manifest);
// Only inspect the keys we care about.
// Invalidate the cache based off the private _packageCache key as well.
let keys = [].concat(DEPENDENCY_KEYS, '_packageCache');
let key, before, after;
for (let i = 0; i < keys.length; i++) {
key = keys[i];
// Empty keys are identical to undefined keys.
before = stableStringify(parsedCached[key]) || '{}';
after = stableStringify(parsedNew[key]) || '{}';
if (before !== after) {
return false;
}
}
return true;
}
/**
* The `_install` method installs the contents of the manifest into the
* specified package cache.
*
* @private
* @method _install
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
*/
_install(label, type) {
this._removeLinks(label, type);
commands[type].invoke('install', { cwd: this.dirs[label] });
this._restoreLinks(label, type);
// If we just did a clean install we can treat it as up-to-date.
upgraded[label] = true;
}
/**
* The `_upgrade` method guarantees that the contents of the manifest are
* allowed to drift in a SemVer compatible manner. It ensures that CI is
* always running against the latest versions of all dependencies.
*
* @private
* @method _upgrade
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
*/
_upgrade(label, type) {
// Lock out upgrade calls after the first time upgrading the cache.
if (upgraded[label]) {
return;
}
if (!this._canUpgrade(label, type)) {
// Only way to get repeatable behavior in npm: start over.
// We turn an `_upgrade` task into an `_install` task.
fs.removeSync(path.join(this.dirs[label], translate(type, 'path')));
return this._install(label, type);
}
this._removeLinks(label, type);
commands[type].invoke(translate(type, 'upgrade'), { cwd: this.dirs[label] });
this._restoreLinks(label, type);
upgraded[label] = true;
}
_canUpgrade(label, type) {
return type === 'bower' || (type === 'yarn' && fs.existsSync(path.join(this.dirs[label], 'yarn.lock')));
}
// PUBLIC API BELOW
/**
* The `create` method adds a new package cache entry.
*
* @method create
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
* @param {String} manifest The contents of the manifest file for the cache.
* @param {Array} links Packages to omit for install and link.
* @return {String} The directory on disk which contains the cache.
*/
create(label, type, manifest, links) {
links = links || [];
// Save metadata about the PackageCache invocation in the manifest.
let packageManagerVersion = commands[type].invoke('--version').stdout;
let jsonManifest = JSON.parse(manifest);
jsonManifest._packageCache = {
node: process.version,
packageManager: type,
packageManagerVersion,
links,
};
manifest = JSON.stringify(jsonManifest);
// Compare any existing manifest to the ideal per current blueprint.
let identical = this._checkManifest(label, type, manifest);
if (identical) {
// Use what we have, but opt in to SemVer drift.
this._upgrade(label, type);
} else {
// Tell the package manager to start semi-fresh.
this._writeManifest(label, type, manifest);
this._install(label, type);
}
return this.dirs[label];
}
/**
* The `update` method aliases the `create` method.
*
* @method update
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
* @param {String} manifest The contents of the manifest file for the cache.
* @param {Array} links Packages to elide for install and link.
* @return {String} The directory on disk which contains the cache.
*/
update(/*label, type, manifest, links*/) {
return this.create.apply(this, arguments);
}
/**
* The `get` method returns the directory for the cache.
*
* @method get
* @param {String} label The label for the cache.
* @return {String} The directory on disk which contains the cache.
*/
get(label) {
return this.dirs[label];
}
/**
* The `destroy` method removes all evidence of the package cache.
*
* @method destroy
* @param {String} label The label for the cache.
* @param {String} type The type of package cache.
*/
destroy(label) {
process.chdir(this.rootPath);
quickTemp.remove(this.dirs, label);
process.chdir(originalWorkingDirectory);
this._conf.delete(label);
}
/**
* The `clone` method duplicates a cache. Some package managers can
* leverage a pre-existing state to speed up their installation.
*
* @method destroy
* @param {String} fromLabel The label for the cache to clone.
* @param {String} toLabel The label for the new cache.
*/
clone(fromLabel, toLabel) {
process.chdir(this.rootPath);
let outputDir = quickTemp.makeOrReuse(this.dirs, toLabel);
process.chdir(originalWorkingDirectory);
this._conf.set(toLabel, outputDir);
fs.copySync(this.get(fromLabel), outputDir);
return this.dirs[toLabel];
}
};