From 6607cce8f0aecff9c49aeaeac2722dc75cd771fb Mon Sep 17 00:00:00 2001 From: Haruaki OTAKE Date: Fri, 10 Apr 2020 19:57:36 +0900 Subject: [PATCH 1/2] refactor: new cache implementation add Filesystem Cache implementation instead of loader-fs-cache --- package-lock.json | 102 ++++++++++++++----------- package.json | 2 +- src/cache.js | 187 +++++++++++++++++++++++++++++++++++++++++++++ src/cacheLoader.js | 45 +++++------ test/cache.test.js | 4 +- 5 files changed, 267 insertions(+), 73 deletions(-) create mode 100644 src/cache.js diff --git a/package-lock.json b/package-lock.json index fa89d5e..b0df449 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5672,47 +5672,78 @@ } }, "find-cache-dir": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-0.1.1.tgz", - "integrity": "sha1-yN765XyKUqinhPnjHFfHQumToLk=", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.1.tgz", + "integrity": "sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ==", "requires": { "commondir": "^1.0.1", - "mkdirp": "^0.5.1", - "pkg-dir": "^1.0.0" + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" }, "dependencies": { "find-up": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", - "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "requires": { - "path-exists": "^2.0.0", - "pinkie-promise": "^2.0.0" + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" } }, - "mkdirp": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.4.tgz", - "integrity": "sha512-iG9AK/dJLtJ0XNgTuDbSyNS3zECqDlAhnQW4CsNxBG3LQJBbHmRX1egw39DmtOdCAqY+dKXV+sgPgilNWUKMVw==", + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "requires": { - "minimist": "^1.2.5" + "p-locate": "^4.1.0" } }, - "path-exists": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", - "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "make-dir": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.2.tgz", + "integrity": "sha512-rYKABKutXa6vXTXhoV18cBE7PaewPXHe/Bdq4v+ZLMhxbWApkFFplT0LcbMW+6BbjnQXzZ/sAvSE/JdguApG5w==", "requires": { - "pinkie-promise": "^2.0.0" + "semver": "^6.0.0" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" } }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "requires": { + "p-limit": "^2.2.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, "pkg-dir": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-1.0.0.tgz", - "integrity": "sha1-ektQio1bstYp1EcFb/TpyTFM89Q=", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", "requires": { - "find-up": "^1.0.0" + "find-up": "^4.0.0" } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" } } }, @@ -10152,25 +10183,6 @@ } } }, - "loader-fs-cache": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/loader-fs-cache/-/loader-fs-cache-1.0.3.tgz", - "integrity": "sha512-ldcgZpjNJj71n+2Mf6yetz+c9bM4xpKtNds4LbqXzU/PTdeAX0g3ytnU1AJMEcTk2Lex4Smpe3Q/eCTsvUBxbA==", - "requires": { - "find-cache-dir": "^0.1.1", - "mkdirp": "^0.5.1" - }, - "dependencies": { - "mkdirp": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.4.tgz", - "integrity": "sha512-iG9AK/dJLtJ0XNgTuDbSyNS3zECqDlAhnQW4CsNxBG3LQJBbHmRX1egw39DmtOdCAqY+dKXV+sgPgilNWUKMVw==", - "requires": { - "minimist": "^1.2.5" - } - } - } - }, "loader-runner": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.4.0.tgz", @@ -11363,12 +11375,14 @@ "pinkie": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", - "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=", + "dev": true }, "pinkie-promise": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "dev": true, "requires": { "pinkie": "^2.0.0" } diff --git a/package.json b/package.json index 9777bab..c64458e 100644 --- a/package.json +++ b/package.json @@ -41,8 +41,8 @@ "webpack": "^4.0.0 || ^5.0.0" }, "dependencies": { + "find-cache-dir": "^3.3.1", "fs-extra": "^9.0.0", - "loader-fs-cache": "^1.0.3", "loader-utils": "^2.0.0", "object-hash": "^2.0.3", "schema-utils": "^2.6.5" diff --git a/src/cache.js b/src/cache.js new file mode 100644 index 0000000..f6aaa9f --- /dev/null +++ b/src/cache.js @@ -0,0 +1,187 @@ +/** + * Original Filesystem Cache implementation by babel-loader + * Licensed under the MIT License + * + * @see https://github.com/babel/babel-loader/commits/master/src/fs-cache.js + * @see https://github.com/babel/babel-loader/commits/master/src/cache.js + */ + +/** + * Filesystem Cache + * + * Given a file and a transform function, cache the result into files + * or retrieve the previously cached files if the given file is already known. + * + * @see https://github.com/babel/babel-loader/issues/34 + * @see https://github.com/babel/babel-loader/pull/41 + */ +import fs from 'fs'; +import os from 'os'; +import { join } from 'path'; +import { promisify } from 'util'; +import zlib from 'zlib'; +import { createHash } from 'crypto'; + +import findCacheDir from 'find-cache-dir'; + +// Lazily instantiated when needed +let defaultCacheDirectory = null; + +const readFile = promisify(fs.readFile); +const writeFile = promisify(fs.writeFile); +const gunzip = promisify(zlib.gunzip); +const gzip = promisify(zlib.gzip); + +/** + * Read the contents from the compressed file. + * + * @async + * @params {String} filename + * @params {Boolean} compress + */ +const read = async (filename, compress) => { + const data = await readFile(filename + (compress ? '.gz' : '')); + const content = compress ? await gunzip(data) : data; + + return JSON.parse(content.toString()); +}; + +/** + * Write contents into a compressed file. + * + * @async + * @params {String} filename + * @params {Boolean} compress + * @params {String} result + */ +const write = async (filename, compress, result) => { + const content = JSON.stringify(result); + + const data = compress ? await gzip(content) : content; + return writeFile(filename + (compress ? '.gz' : ''), data); +}; + +/** + * Build the filename for the cached file + * + * @params {String} source File source code + * @params {String} identifier + * @params {Object} options Options used + * + * @return {String} + */ +const filename = (source, identifier, options) => { + const hash = createHash('md4'); + + const contents = JSON.stringify({ source, options, identifier }); + + hash.update(contents); + + return `${hash.digest('hex')}.json`; +}; + +/** + * Handle the cache + * + * @params {String} directory + * @params {Object} params + */ +const handleCache = async (directory, params) => { + const { + source, + options = {}, + transform, + cacheIdentifier, + cacheDirectory, + cacheCompression, + } = params; + + const file = join(directory, filename(source, cacheIdentifier, options)); + + try { + // No errors mean that the file was previously cached + // we just need to return it + return await read(file, cacheCompression); + // eslint-disable-next-line no-empty + } catch (err) {} + + const fallback = + typeof cacheDirectory !== 'string' && directory !== os.tmpdir(); + + // Make sure the directory exists. + try { + fs.mkdirSync(directory, { recursive: true }); + } catch (err) { + if (fallback) { + return handleCache(os.tmpdir(), params); + } + + throw err; + } + + // Otherwise just transform the file + // return it to the user asap and write it in cache + const result = await transform(source, options); + + try { + await write(file, cacheCompression, result); + } catch (err) { + if (fallback) { + // Fallback to tmpdir if node_modules folder not writable + return handleCache(os.tmpdir(), params); + } + + throw err; + } + + return result; +}; + +/** + * Retrieve file from cache, or create a new one for future reads + * + * @async + * @param {Object} params + * @param {String} params.cacheDirectory Directory to store cached files + * @param {String} params.cacheIdentifier Unique identifier to bust cache + * @param {Boolean} params.cacheCompression + * @param {String} params.source Original contents of the file to be cached + * @param {Object} params.options Options to be given to the transform fn + * @param {Function} params.transform Function that will transform the + * original file and whose result will be + * cached + * + * @example + * + * cache({ + * cacheDirectory: '.tmp/cache', + * cacheIdentifier: 'babel-loader-cachefile', + * cacheCompression: true, + * source: *source code from file*, + * options: { + * experimental: true, + * runtime: true + * }, + * transform: function(source, options) { + * var content = *do what you need with the source* + * return content; + * } + * }); + */ + +module.exports = async (params) => { + let directory; + + if (typeof params.cacheDirectory === 'string') { + directory = params.cacheDirectory; + } else { + if (defaultCacheDirectory === null) { + defaultCacheDirectory = + findCacheDir({ name: 'eslint-loader' }) || os.tmpdir(); + } + + directory = defaultCacheDirectory; + } + + return handleCache(directory, params); +}; diff --git a/src/cacheLoader.js b/src/cacheLoader.js index b09bbdd..7672e32 100644 --- a/src/cacheLoader.js +++ b/src/cacheLoader.js @@ -1,8 +1,6 @@ -import createCache from 'loader-fs-cache'; - import { version } from '../package.json'; -const cache = createCache('eslint-loader'); +import cache from './cache'; export default function cacheLoader(linter, content, map) { const { loaderContext, options, CLIEngine } = linter; @@ -12,31 +10,26 @@ export default function cacheLoader(linter, content, map) { eslint: CLIEngine.version, }); - cache( - { - directory: options.cache, - identifier: cacheIdentifier, - options, - source: content, - transform() { - return linter.lint(content); - }, + cache({ + cacheDirectory: options.cache, + cacheIdentifier, + cacheCompression: true, + options, + source: content, + transform() { + return linter.lint(content); }, - (err, res) => { - // istanbul ignore next - if (err) { - return callback(err); - } - - let error = err; - + }) + .then((res) => { try { linter.printOutput({ ...res, src: content }); - } catch (e) { - error = e; + } catch (error) { + return callback(error, content, map); } - - return callback(error, content, map); - } - ); + return callback(null, content, map); + }) + .catch((err) => { + // istanbul ignore next + return callback(err); + }); } diff --git a/test/cache.test.js b/test/cache.test.js index 61ab1f8..4cd01d5 100644 --- a/test/cache.test.js +++ b/test/cache.test.js @@ -1,7 +1,7 @@ import { join } from 'path'; +import fs from 'fs'; import { readdirSync, removeSync } from 'fs-extra'; -import mkdirp from 'mkdirp'; import webpack from 'webpack'; const defaultCacheDir = join(__dirname, '../node_modules/.cache/eslint-loader'); @@ -25,7 +25,7 @@ function createTestDirectory(dir) { const directory = join(dir, 'cache'); removeSync(directory); - mkdirp.sync(directory); + fs.mkdirSync(directory, { recursive: true }); return directory; } From acdae3827d7d6ed85393c3c539df9f0f181efd10 Mon Sep 17 00:00:00 2001 From: Haruaki OTAKE Date: Fri, 10 Apr 2020 20:30:56 +0900 Subject: [PATCH 2/2] chore: remove unused devDependencies --- package-lock.json | 6 ------ package.json | 1 - 2 files changed, 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index b0df449..62b6a0e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10640,12 +10640,6 @@ } } }, - "mkdirp": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.3.tgz", - "integrity": "sha512-6uCP4Qc0sWsgMLy1EOqqS/3rjDHOEnsStVr/4vtAIK2Y5i2kA7lFFejYrpIyiN9w0pYf4ckeCYT9f1r1P9KX5g==", - "dev": true - }, "modify-values": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/modify-values/-/modify-values-1.0.1.tgz", diff --git a/package.json b/package.json index c64458e..5a5fa5d 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,6 @@ "jest": "^25.2.6", "jest-junit": "^10.0.0", "lint-staged": "^10.1.1", - "mkdirp": "^1.0.3", "npm-run-all": "^4.1.5", "prettier": "^2.0.2", "standard-version": "^7.1.0",