diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c7513eb405..8d8f80c958 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,9 +49,28 @@ jobs: - run: npm install - run: npm run run-rules-on-codebase integration: + name: Integration test (${{ matrix.group }}) + strategy: + fail-fast: false + matrix: + group: + - "1" + - "2" + - "3" + - "4" + - "5" + - "6" + - "7" + - "8" + - "9" + - "10" + - "11" + - "12" + env: + TIMING: 1 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 - run: npm install - - run: npm run integration + - run: npm run integration -- --group ${{ matrix.group }} diff --git a/package.json b/package.json index b384bf2523..08fd891f55 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,8 @@ "outdent": "^0.8.0", "typescript": "^4.8.3", "vue-eslint-parser": "^9.1.0", - "xo": "^0.52.3" + "xo": "^0.52.3", + "yaml": "^1.10.2" }, "peerDependencies": { "eslint": ">=8.23.1" diff --git a/test/integration/config.js b/test/integration/config.js deleted file mode 100644 index d9af78bf98..0000000000 --- a/test/integration/config.js +++ /dev/null @@ -1,38 +0,0 @@ -'use strict'; - -module.exports = { - root: true, - parser: '@babel/eslint-parser', - parserOptions: { - requireConfigFile: false, - babelOptions: { - babelrc: false, - configFile: false, - parserOpts: { - plugins: [ - 'jsx', - 'doExpressions', - 'exportDefaultFrom', - ], - }, - }, - }, - plugins: [ - 'unicorn', - ], - extends: 'plugin:unicorn/all', - rules: { - // This rule crashing on replace string inside `jsx` or `Unicode escape sequence` - 'unicorn/string-content': 'off', - }, - overrides: [ - { - files: ['*.ts'], - parser: '@typescript-eslint/parser', - }, - { - files: ['*.vue'], - parser: 'vue-eslint-parser', - }, - ], -}; diff --git a/test/integration/package.json b/test/integration/package.json deleted file mode 100644 index ae5df13bef..0000000000 --- a/test/integration/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "private": true, - "name": "eslint-config-unicorn-tester", - "engines": { - "node": ">=12" - }, - "dependencies": { - "eslint-plugin-unicorn": "file:../.." - } -} diff --git a/test/integration/projects.mjs b/test/integration/projects.mjs index d1ee160bf9..7599fc37ee 100644 --- a/test/integration/projects.mjs +++ b/test/integration/projects.mjs @@ -3,64 +3,125 @@ import {fileURLToPath} from 'node:url'; const dirname = path.dirname(fileURLToPath(import.meta.url)); +function normalizeProject(project) { + if (typeof project === 'string') { + project = {repository: project}; + } + + const { + repository, + name = repository.split('/').pop(), + ignore = [], + } = project; + + return { + location: path.join(dirname, 'fixtures', name), + ...project, + name, + repository, + ignore, + }; +} + export default [ - { - name: 'fixtures-local', - location: path.join(dirname, 'fixtures-local'), - }, - { - repository: 'https://github.com/avajs/ava', - ignore: [ - 'test/node_modules', - ], - }, - 'https://github.com/chalk/chalk', - 'https://github.com/chalk/wrap-ansi', - 'https://github.com/sindresorhus/np', - 'https://github.com/sindresorhus/ora', - 'https://github.com/sindresorhus/p-map', - 'https://github.com/sindresorhus/os-locale', - 'https://github.com/sindresorhus/execa', - 'https://github.com/sindresorhus/pify', - 'https://github.com/sindresorhus/boxen', - 'https://github.com/sindresorhus/make-dir', - 'https://github.com/SamVerschueren/listr', - 'https://github.com/SamVerschueren/listr-update-renderer', - 'https://github.com/SamVerschueren/clinton', - 'https://github.com/SamVerschueren/bragg', - 'https://github.com/SamVerschueren/bragg-router', - 'https://github.com/SamVerschueren/dev-time', - 'https://github.com/SamVerschueren/decode-uri-component', - 'https://github.com/kevva/to-ico', - 'https://github.com/kevva/download', - 'https://github.com/kevva/brightness', - 'https://github.com/kevva/decompress', - 'https://github.com/kevva/npm-conf', - 'https://github.com/imagemin/imagemin', - 'https://github.com/qix-/color-convert', - 'https://github.com/sindresorhus/ky', - 'https://github.com/sindresorhus/query-string', - 'https://github.com/sindresorhus/meow', - 'https://github.com/sindresorhus/globby', - 'https://github.com/sindresorhus/emittery', - 'https://github.com/sindresorhus/p-queue', - 'https://github.com/sindresorhus/pretty-bytes', - 'https://github.com/sindresorhus/normalize-url', - 'https://github.com/sindresorhus/pageres', - { - repository: 'https://github.com/sindresorhus/got', - ignore: [ - // This file use `package` keyword as variable - 'documentation/examples/gh-got.js', - ], - }, + [ + { + name: 'fixtures-local', + location: path.join(dirname, 'fixtures-local'), + }, + { + repository: 'https://github.com/avajs/ava', + ignore: [ + 'test/node_modules', + 'test-tap/fixture/report/edgecases/ast-syntax-error.cjs', + ], + }, + 'https://github.com/chalk/chalk', + 'https://github.com/chalk/wrap-ansi', + 'https://github.com/sindresorhus/np', + 'https://github.com/sindresorhus/ora', + 'https://github.com/sindresorhus/p-map', + 'https://github.com/sindresorhus/os-locale', + 'https://github.com/sindresorhus/execa', + 'https://github.com/sindresorhus/pify', + 'https://github.com/sindresorhus/boxen', + 'https://github.com/sindresorhus/make-dir', + 'https://github.com/sindresorhus/ky', + 'https://github.com/sindresorhus/query-string', + 'https://github.com/sindresorhus/meow', + 'https://github.com/sindresorhus/globby', + 'https://github.com/sindresorhus/emittery', + 'https://github.com/sindresorhus/p-queue', + 'https://github.com/sindresorhus/pretty-bytes', + 'https://github.com/sindresorhus/normalize-url', + 'https://github.com/sindresorhus/pageres', + { + repository: 'https://github.com/sindresorhus/got', + ignore: [ + // This file use `package` keyword as variable + 'documentation/examples/gh-got.js', + ], + }, + 'https://github.com/sindresorhus/create-dmg', + 'https://github.com/sindresorhus/cp-file', + 'https://github.com/sindresorhus/capture-website', + { + repository: 'https://github.com/sindresorhus/file-type', + ignore: [ + // Contains non-text `.mts` file + 'fixture/**', + ], + }, + 'https://github.com/sindresorhus/slugify', + 'https://github.com/SamVerschueren/listr', + 'https://github.com/SamVerschueren/listr-update-renderer', + 'https://github.com/SamVerschueren/clinton', + 'https://github.com/SamVerschueren/bragg', + 'https://github.com/SamVerschueren/bragg-router', + 'https://github.com/SamVerschueren/dev-time', + 'https://github.com/SamVerschueren/decode-uri-component', + 'https://github.com/kevva/to-ico', + 'https://github.com/kevva/download', + 'https://github.com/kevva/brightness', + 'https://github.com/kevva/decompress', + 'https://github.com/kevva/npm-conf', + 'https://github.com/imagemin/imagemin', + 'https://github.com/qix-/color-convert', + { + repository: 'https://github.com/prettier/prettier', + ignore: [ + 'tests/**', + ], + }, + { + repository: 'https://github.com/puppeteer/puppeteer', + ignore: [ + // Parser error on `await page.evaluate(() => delete Node);` + // https://github.com/puppeteer/puppeteer/blob/0b1a9ceee2f05f534f0d50079ece172d627a93c7/test/jshandle.spec.js#L151 + 'test/jshandle.spec.js', + + // `package` keyword + // https://github.com/puppeteer/puppeteer/blob/0b1a9ceee2f05f534f0d50079ece172d627a93c7/utils/apply_next_version.js#L17 + 'utils/apply_next_version.js', + + // Global return + 'utils/fetch_devices.js', + ], + }, + 'https://github.com/ReactTraining/react-router', + // #902 + { + repository: 'https://github.com/reakit/reakit', + ignore: [ + 'packages/reakit/jest.config.js', // This file use `package` keyword as variable + ], + }, + // #1030 + 'https://github.com/astrofox-io/astrofox', + // #1075 + 'https://github.com/jaredLunde/masonic', + ], // 'https://github.com/eslint/eslint', - { - repository: 'https://github.com/prettier/prettier', - ignore: [ - 'tests/**', - ], - }, { repository: 'https://github.com/angular/angular', ignore: [ @@ -79,42 +140,20 @@ export default [ 'build/**', ], }, - // This repo use `override` keyword which is not avaiable before TS4.3, temporary disable - // https://github.com/microsoft/vscode/pull/120690/files - // { - // repository: 'https://github.com/microsoft/vscode', - // ignore: [ - // // This file use `'\033'` - // 'build/**' - // ] - // }, - // 'https://github.com/ElemeFE/element', - // 'https://github.com/iview/iview', - 'https://github.com/sindresorhus/create-dmg', - 'https://github.com/sindresorhus/cp-file', - 'https://github.com/sindresorhus/capture-website', - 'https://github.com/sindresorhus/file-type', - 'https://github.com/sindresorhus/slugify', { - repository: 'https://github.com/gatsbyjs/gatsby', + repository: 'https://github.com/microsoft/vscode', ignore: [ - // These files use `flow` - '**/*.js', + // This file use `'\033'` + 'build/**', ], }, + 'https://github.com/element-plus/element-plus', + 'https://github.com/tusen-ai/naive-ui', { - repository: 'https://github.com/puppeteer/puppeteer', + repository: 'https://github.com/gatsbyjs/gatsby', ignore: [ - // Parser error on `await page.evaluate(() => delete Node);` - // https://github.com/puppeteer/puppeteer/blob/0b1a9ceee2f05f534f0d50079ece172d627a93c7/test/jshandle.spec.js#L151 - 'test/jshandle.spec.js', - - // `package` keyword - // https://github.com/puppeteer/puppeteer/blob/0b1a9ceee2f05f534f0d50079ece172d627a93c7/utils/apply_next_version.js#L17 - 'utils/apply_next_version.js', - - // Global return - 'utils/fetch_devices.js', + // These files use `flow` + '**/*.js', ], }, { @@ -132,7 +171,6 @@ export default [ 'scripts/create-package.js', // This file use `package` keyword as variable ], }, - 'https://github.com/ReactTraining/react-router', 'https://github.com/mozilla/pdf.js', // #912 { @@ -146,37 +184,13 @@ export default [ 'scripts/cypress.js', ], }, - // #902 - { - repository: 'https://github.com/reakit/reakit', - ignore: [ - 'packages/reakit/jest.config.js', // This file use `package` keyword as variable - ], - }, // #903 'https://github.com/mattermost/mattermost-webapp', - // #1030 - 'https://github.com/astrofox-io/astrofox', - // #1075 - 'https://github.com/jaredLunde/masonic', // These two project use `decorator`, try to enable when we use `@babel/eslint-parser` // 'https://github.com/untitled-labs/metabase-custom', // 'https://github.com/TheThingsNetwork/lorawan-stack', -].map(project => { - if (typeof project === 'string') { - project = {repository: project}; - } - - const { - repository, - name = repository.split('/').pop(), - ignore = [], - } = project; - - return { - ...project, - name, - repository, - ignore, - }; -}); +].flatMap((projectOrProjects, index) => + Array.isArray(projectOrProjects) + ? projectOrProjects.map(project => ({...normalizeProject(project), group: index})) + : [{...normalizeProject(projectOrProjects), group: index}], +); diff --git a/test/integration/readme.md b/test/integration/readme.md index 63931ce6aa..3494c13647 100644 --- a/test/integration/readme.md +++ b/test/integration/readme.md @@ -2,4 +2,4 @@ To run the integration tests, go to the project root, and run `$ npm run integration`. -To run tests on specific projects, run `$ npm run integration projectName1 projectName2 … projectNameN`. The project names can be found in [`projects.js`](projects.js). +To run tests on specific projects, run `$ npm run integration projectName1 projectName2 … projectNameN`. The project names can be found in [`projects.mjs`](projects.mjs). diff --git a/test/integration/run-eslint.mjs b/test/integration/run-eslint.mjs new file mode 100644 index 0000000000..f70a9beae1 --- /dev/null +++ b/test/integration/run-eslint.mjs @@ -0,0 +1,127 @@ +import {codeFrameColumns} from '@babel/code-frame'; +import {ESLint} from 'eslint'; +import chalk from 'chalk'; +import {outdent} from 'outdent'; +import eslintPluginUnicorn from '../../index.js'; + +class UnicornIntegrationTestError extends AggregateError { + name = 'UnicornIntegrationTestError'; + + constructor(project, errors) { + super(errors, `Error thrown when linting '${project.name}' project.`); + + this.project = project; + } +} + +class UnicornEslintFatalError extends SyntaxError { + name = 'UnicornEslintFatalError'; + + constructor(message, file) { + super(message.message); + + this.eslintMessage = message; + this.eslintFile = file; + } + + get codeFrame() { + const {source, output} = this.eslintFile; + const {line, column, message, ruleId} = this.eslintMessage; + + return codeFrameColumns( + source ?? output, + {start: {line, column}}, + { + message: ruleId ? `[${ruleId}]: ${message}` : message, + highlightCode: true, + }, + ); + } +} + +const sum = (collection, fieldName) => + collection.reduce((total, {[fieldName]: value}) => total + value, 0); + +async function runEslint(project) { + const eslint = new ESLint({ + cwd: project.location, + baseConfig: eslintPluginUnicorn.configs.all, + useEslintrc: false, + extensions: ['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx', '.vue'], + plugins: { + unicorn: eslintPluginUnicorn, + }, + fix: true, + overrideConfig: { + parser: '@babel/eslint-parser', + parserOptions: { + requireConfigFile: false, + babelOptions: { + babelrc: false, + configFile: false, + parserOpts: { + allowReturnOutsideFunction: true, + plugins: [ + 'jsx', + 'exportDefaultFrom', + ], + }, + }, + }, + ignorePatterns: project.ignore, + rules: { + // This rule crashing on replace string inside `jsx` or `Unicode escape sequence` + 'unicorn/string-content': 'off', + }, + overrides: [ + { + files: ['*.ts', '*.mts', '*.cts', '*.tsx'], + parser: '@typescript-eslint/parser', + parserOptions: { + project: [], + }, + }, + { + files: ['*.vue'], + parser: 'vue-eslint-parser', + parserOptions: { + parser: '@typescript-eslint/parser', + ecmaFeatures: { + jsx: true, + }, + project: [], + }, + }, + ], + }, + }); + + const results = await eslint.lintFiles('.'); + + const errors = results + .filter(file => file.fatalErrorCount > 0) + .flatMap( + file => file.messages + .filter(message => message.fatal) + .map(message => new UnicornEslintFatalError(message, file)), + ); + + if (errors.length > 0) { + throw new UnicornIntegrationTestError(project, errors); + } + + const errorCount = sum(results, 'errorCount'); + const warningCount = sum(results, 'warningCount'); + const fixableErrorCount = sum(results, 'fixableErrorCount'); + const fixableWarningCount = sum(results, 'fixableWarningCount'); + console.log(); + console.log(outdent` + ${chalk.green.bold.underline(`[${project.name}]`)} ${results.length} files linted: + - error: ${chalk.gray(errorCount)} + - warning: ${chalk.gray(warningCount)} + - fixable error: ${chalk.gray(fixableErrorCount)} + - fixable warning: ${chalk.gray(fixableWarningCount)} + `); +} + +export default runEslint; diff --git a/test/integration/test.mjs b/test/integration/test.mjs index ba29cbbc4c..7e687ad2d3 100644 --- a/test/integration/test.mjs +++ b/test/integration/test.mjs @@ -2,186 +2,150 @@ import process from 'node:process'; import fs from 'node:fs'; import path from 'node:path'; -import {fileURLToPath} from 'node:url'; +import {parseArgs} from 'node:util'; import Listr from 'listr'; import {execa} from 'execa'; import chalk from 'chalk'; +import {outdent} from 'outdent'; import {isCI} from 'ci-info'; import mem from 'mem'; +import YAML from 'yaml'; import allProjects from './projects.mjs'; - -const dirname = path.dirname(fileURLToPath(import.meta.url)); -const projectsArguments = process.argv.slice(2); -const projects = projectsArguments.length === 0 - ? allProjects - : allProjects.filter(({name}) => projectsArguments.includes(name)); - -const enrichErrors = (packageName, cliArguments, f) => async (...arguments_) => { - try { - return await f(...arguments_); - } catch (error) { - error.packageName = packageName; - error.cliArgs = cliArguments; - throw error; - } -}; - -const makeEslintTask = (project, destination) => { - const arguments_ = [ - 'eslint', - project.path || '.', - '--fix-dry-run', - '--no-eslintrc', - '--ext', - '.js,.ts,.vue', - '--format', - 'json', - '--config', - path.join(dirname, 'config.js'), - ]; - - for (const pattern of project.ignore) { - arguments_.push('--ignore-pattern', pattern); +import runEslint from './run-eslint.mjs'; + +if (isCI) { + const CI_CONFIG_FILE = new URL('../../.github/workflows/main.yml', import.meta.url); + const content = fs.readFileSync(CI_CONFIG_FILE, 'utf8'); + const config = YAML.parse(content).jobs.integration.strategy.matrix.group; + + const expected = [...new Set(allProjects.map(project => String(project.group + 1)))]; + if ( + config.length !== expected.length + || expected.some((group, index) => config[index] !== group) + ) { + throw new Error(outdent` + Expect 'jobs.integration.strategy.matrix.group' in '/.github/workflows/main.yml' to be: + ${YAML.stringify(expected)} + `); } +} - return enrichErrors(project.name, arguments_, async () => { - let stdout; - let processError; - try { - ({stdout} = await execa('npx', arguments_, {cwd: destination, localDir: dirname})); - } catch (error) { - ({stdout} = error); - processError = error; - - if (!stdout) { - throw error; - } - } +const { + values: { + group, + }, + positionals: projectsArguments, +} = parseArgs({ + options: { + group: { + type: 'string', + }, + }, + allowPositionals: true, +}); - let files; - try { - files = JSON.parse(stdout); - } catch (error) { - console.error('Error while parsing eslint output:', error); +let projects = projectsArguments.length === 0 + ? allProjects + : allProjects.filter(({name}) => projectsArguments.includes(name)); - if (processError) { - throw processError; - } +if (isCI && !group) { + throw new Error('"--group" is required'); +} - throw error; - } +if (group) { + projects = projects.filter(project => String(project.group + 1) === group); +} - for (const file of files) { - for (const message of file.messages) { - if (message.fatal) { - const error = new Error(message.message); - error.eslintJob = { - destination, - project, - file, - }; - error.eslintMessage = message; - throw error; - } - } - } - }); -}; +if (projects.length === 0) { + console.log('No project matched'); + process.exit(0); +} const getBranch = mem(async dirname => { const {stdout} = await execa('git', ['branch', '--show-current'], {cwd: dirname}); return stdout; }); -const execute = project => { - const destination = project.location || path.join(dirname, 'fixtures', project.name); - - return new Listr([ +const execute = project => new Listr( + [ { title: 'Cloning', - skip: () => fs.existsSync(destination) ? 'Project already downloaded.' : false, + skip: () => fs.existsSync(project.location) ? 'Project already downloaded.' : false, task: () => execa('git', [ 'clone', project.repository, '--single-branch', '--depth', '1', - destination, - ]), + project.location, + ], {stdout: 'inherit', stderr: 'inherit'}), }, { title: 'Running eslint', - task: makeEslintTask(project, destination), + task: () => runEslint(project), }, ].map(({title, task, skip}) => ({ title: `${project.name} / ${title}`, skip, task, - })), { - exitOnError: false, - }); -}; - -const list = new Listr([ - { - title: 'Setup', - task: () => execa('npm', ['install'], {cwd: dirname, stdout: 'inherit', stderr: 'inherit'}), - }, - { - title: 'Integration tests', - task() { - const tests = new Listr({concurrent: true}); - - for (const project of projects) { - tests.add([ - { - title: project.name, - task: () => execute(project), - }, - ]); - } - - return tests; - }, - }, -], { - renderer: isCI ? 'verbose' : 'default', -}); + })), + {exitOnError: false}, +); + +async function printEslintError(eslintError) { + const {message, project} = eslintError; + + console.log(); + console.error( + chalk.red.bold.underline(`[${project.name}]`), + message, + ); + + project.branch ??= await getBranch(project.location); + for (const error of eslintError.errors) { + let file = path.relative(project.location, error.eslintFile.filePath); + if (project.repository) { + file = `${project.repository}/blob/${project.branch}/${file}`; + } -async function logError(error) { - if (error.errors) { - for (const error2 of error.errors) { - console.error('\n', chalk.red.bold.underline(error2.packageName), chalk.gray('(' + error2.cliArgs.join(' ') + ')')); - console.error(error2.message); - - if (error2.stderr) { - console.error(chalk.gray(error2.stderr)); - } - - if (error2.eslintMessage) { - const {file, project, destination} = error2.eslintJob; - const {line} = error2.eslintMessage; - - if (project.repository) { - // eslint-disable-next-line no-await-in-loop - const branch = await getBranch(destination); - console.error(chalk.gray(`${project.repository}/blob/${branch}/${path.relative(destination, file.filePath)}#L${line}`)); - } else { - console.error(chalk.gray(`${path.relative(destination, file.filePath)}#L${line}`)); - } - - console.error(chalk.gray(JSON.stringify(error2.eslintMessage, undefined, 2))); - } + if (typeof error.eslintMessage.line === 'number') { + file += `#L${error.eslintMessage.line}`; } - } else { - console.error(error); + + console.log(); + console.error(chalk.blue.bold.underline(file)); + console.log(); + console.error(error.codeFrame); } +} - process.exit(1); +async function printListrError(listrError) { + process.exitCode = 1; + + if (!listrError.errors) { + console.error(listrError); + return; + } + + for (const error of listrError.errors) { + if (error.name !== 'UnicornIntegrationTestError') { + console.error(error); + continue; + } + + // eslint-disable-next-line no-await-in-loop + await printEslintError(error); + } } try { - await list.run(); + await new Listr( + projects.map(project => ({title: project.name, task: () => execute(project)})), + { + renderer: isCI ? 'verbose' : 'default', + concurrent: true, + }, + ).run(); } catch (error) { - await logError(error); + await printListrError(error); } diff --git a/test/smoke/eslint-remote-tester.config.js b/test/smoke/eslint-remote-tester.config.js index 93b39a0330..7dbfba9971 100644 --- a/test/smoke/eslint-remote-tester.config.js +++ b/test/smoke/eslint-remote-tester.config.js @@ -29,6 +29,7 @@ module.exports = { ecmaFeatures: { jsx: true, }, + project: [], }, extends: ['plugin:unicorn/all'], }, diff --git a/test/utils/parsers.mjs b/test/utils/parsers.mjs index b63e7ba04c..6f1e4cb959 100644 --- a/test/utils/parsers.mjs +++ b/test/utils/parsers.mjs @@ -48,6 +48,7 @@ const typescript = { mergeParserOptions(options) { return { ...defaultOptions.parserOptions, + project: [], ...options, }; },