New integration test

.github/workflows/main.yml

@@ -49,9 +49,28 @@ jobs:
       - run: npm install
       - run: npm run run-rules-on-codebase
   integration:
+    name: Integration test (${{ matrix.group }})
+    strategy:
+      fail-fast: false
+      matrix:
+        group:
+          - "1"
+          - "2"
+          - "3"
+          - "4"
+          - "5"
+          - "6"
+          - "7"
+          - "8"
+          - "9"
+          - "10"
+          - "11"
+          - "12"
+    env:
+      TIMING: 1
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
       - run: npm install
-      - run: npm run integration
+      - run: npm run integration -- --group ${{ matrix.group }}

package.json

@@ -87,7 +87,8 @@
 		"outdent": "^0.8.0",
 		"typescript": "^4.8.3",
 		"vue-eslint-parser": "^9.1.0",
-		"xo": "^0.52.3"
+		"xo": "^0.52.3",
+		"yaml": "^1.10.2"
 	},
 	"peerDependencies": {
 		"eslint": ">=8.23.1"

test/integration/projects.mjs

@@ -3,64 +3,125 @@ import {fileURLToPath} from 'node:url';
 
 const dirname = path.dirname(fileURLToPath(import.meta.url));
 
+function normalizeProject(project) {
+	if (typeof project === 'string') {
+		project = {repository: project};
+	}
+
+	const {
+		repository,
+		name = repository.split('/').pop(),
+		ignore = [],
+	} = project;
+
+	return {
+		location: path.join(dirname, 'fixtures', name),
+		...project,
+		name,
+		repository,
+		ignore,
+	};
+}
+
 export default [
-	{
-		name: 'fixtures-local',
-		location: path.join(dirname, 'fixtures-local'),
-	},
-	{
-		repository: 'https://github.com/avajs/ava',
-		ignore: [
-			'test/node_modules',
-		],
-	},
-	'https://github.com/chalk/chalk',
-	'https://github.com/chalk/wrap-ansi',
-	'https://github.com/sindresorhus/np',
-	'https://github.com/sindresorhus/ora',
-	'https://github.com/sindresorhus/p-map',
-	'https://github.com/sindresorhus/os-locale',
-	'https://github.com/sindresorhus/execa',
-	'https://github.com/sindresorhus/pify',
-	'https://github.com/sindresorhus/boxen',
-	'https://github.com/sindresorhus/make-dir',
-	'https://github.com/SamVerschueren/listr',
-	'https://github.com/SamVerschueren/listr-update-renderer',
-	'https://github.com/SamVerschueren/clinton',
-	'https://github.com/SamVerschueren/bragg',
-	'https://github.com/SamVerschueren/bragg-router',
-	'https://github.com/SamVerschueren/dev-time',
-	'https://github.com/SamVerschueren/decode-uri-component',
-	'https://github.com/kevva/to-ico',
-	'https://github.com/kevva/download',
-	'https://github.com/kevva/brightness',
-	'https://github.com/kevva/decompress',
-	'https://github.com/kevva/npm-conf',
-	'https://github.com/imagemin/imagemin',
-	'https://github.com/qix-/color-convert',
-	'https://github.com/sindresorhus/ky',
-	'https://github.com/sindresorhus/query-string',
-	'https://github.com/sindresorhus/meow',
-	'https://github.com/sindresorhus/globby',
-	'https://github.com/sindresorhus/emittery',
-	'https://github.com/sindresorhus/p-queue',
-	'https://github.com/sindresorhus/pretty-bytes',
-	'https://github.com/sindresorhus/normalize-url',
-	'https://github.com/sindresorhus/pageres',
-	{
-		repository: 'https://github.com/sindresorhus/got',
-		ignore: [
-			// This file use `package` keyword as variable
-			'documentation/examples/gh-got.js',
-		],
-	},
+	[
+		{
+			name: 'fixtures-local',
+			location: path.join(dirname, 'fixtures-local'),
+		},
+		{
+			repository: 'https://github.com/avajs/ava',
+			ignore: [
+				'test/node_modules',
+				'test-tap/fixture/report/edgecases/ast-syntax-error.cjs',
+			],
+		},
+		'https://github.com/chalk/chalk',
+		'https://github.com/chalk/wrap-ansi',
+		'https://github.com/sindresorhus/np',
+		'https://github.com/sindresorhus/ora',
+		'https://github.com/sindresorhus/p-map',
+		'https://github.com/sindresorhus/os-locale',
+		'https://github.com/sindresorhus/execa',
+		'https://github.com/sindresorhus/pify',
+		'https://github.com/sindresorhus/boxen',
+		'https://github.com/sindresorhus/make-dir',
+		'https://github.com/sindresorhus/ky',
+		'https://github.com/sindresorhus/query-string',
+		'https://github.com/sindresorhus/meow',
+		'https://github.com/sindresorhus/globby',
+		'https://github.com/sindresorhus/emittery',
+		'https://github.com/sindresorhus/p-queue',
+		'https://github.com/sindresorhus/pretty-bytes',
+		'https://github.com/sindresorhus/normalize-url',
+		'https://github.com/sindresorhus/pageres',
+		{
+			repository: 'https://github.com/sindresorhus/got',
+			ignore: [
+				// This file use `package` keyword as variable
+				'documentation/examples/gh-got.js',
+			],
+		},
+		'https://github.com/sindresorhus/create-dmg',
+		'https://github.com/sindresorhus/cp-file',
+		'https://github.com/sindresorhus/capture-website',
+		{
+			repository: 'https://github.com/sindresorhus/file-type',
+			ignore: [
+				// Contains non-text `.mts` file
+				'fixture/**',
+			],
+		},
+		'https://github.com/sindresorhus/slugify',
+		'https://github.com/SamVerschueren/listr',
+		'https://github.com/SamVerschueren/listr-update-renderer',
+		'https://github.com/SamVerschueren/clinton',
+		'https://github.com/SamVerschueren/bragg',
+		'https://github.com/SamVerschueren/bragg-router',
+		'https://github.com/SamVerschueren/dev-time',
+		'https://github.com/SamVerschueren/decode-uri-component',
+		'https://github.com/kevva/to-ico',
+		'https://github.com/kevva/download',
+		'https://github.com/kevva/brightness',
+		'https://github.com/kevva/decompress',
+		'https://github.com/kevva/npm-conf',
+		'https://github.com/imagemin/imagemin',
+		'https://github.com/qix-/color-convert',
+		{
+			repository: 'https://github.com/prettier/prettier',
+			ignore: [
+				'tests/**',
+			],
+		},
+		{
+			repository: 'https://github.com/puppeteer/puppeteer',
+			ignore: [
+				// Parser error on `await page.evaluate(() => delete Node);`
+				// https://github.com/puppeteer/puppeteer/blob/0b1a9ceee2f05f534f0d50079ece172d627a93c7/test/jshandle.spec.js#L151
+				'test/jshandle.spec.js',
+
+				// `package` keyword
+				// https://github.com/puppeteer/puppeteer/blob/0b1a9ceee2f05f534f0d50079ece172d627a93c7/utils/apply_next_version.js#L17
+				'utils/apply_next_version.js',
+
+				// Global return
+				'utils/fetch_devices.js',
+			],
+		},
+		'https://github.com/ReactTraining/react-router',
+		// #902
+		{
+			repository: 'https://github.com/reakit/reakit',
+			ignore: [
+				'packages/reakit/jest.config.js', // This file use `package` keyword as variable
+			],
+		},
+		// #1030
+		'https://github.com/astrofox-io/astrofox',
+		// #1075
+		'https://github.com/jaredLunde/masonic',
+	],
 	// 'https://github.com/eslint/eslint',
-	{
-		repository: 'https://github.com/prettier/prettier',
-		ignore: [
-			'tests/**',
-		],
-	},
 	{
 		repository: 'https://github.com/angular/angular',
 		ignore: [
@@ -79,42 +140,20 @@ export default [
 			'build/**',
 		],
 	},
-	// This repo use `override` keyword which is not avaiable before TS4.3, temporary disable
-	// https://github.com/microsoft/vscode/pull/120690/files
-	// {
-	// 	repository: 'https://github.com/microsoft/vscode',
-	// 	ignore: [
-	// 		// This file use `'\033'`
-	// 		'build/**'
-	// 	]
-	// },
-	// 'https://github.com/ElemeFE/element',
-	// 'https://github.com/iview/iview',
-	'https://github.com/sindresorhus/create-dmg',
-	'https://github.com/sindresorhus/cp-file',
-	'https://github.com/sindresorhus/capture-website',
-	'https://github.com/sindresorhus/file-type',
-	'https://github.com/sindresorhus/slugify',
 	{
-		repository: 'https://github.com/gatsbyjs/gatsby',
+		repository: 'https://github.com/microsoft/vscode',
 		ignore: [
-			// These files use `flow`
-			'**/*.js',
+			// This file use `'\033'`
+			'build/**',
 		],
 	},
+	'https://github.com/element-plus/element-plus',
+	'https://github.com/tusen-ai/naive-ui',
 	{
-		repository: 'https://github.com/puppeteer/puppeteer',
+		repository: 'https://github.com/gatsbyjs/gatsby',
 		ignore: [
-			// Parser error on `await page.evaluate(() => delete Node);`
-			// https://github.com/puppeteer/puppeteer/blob/0b1a9ceee2f05f534f0d50079ece172d627a93c7/test/jshandle.spec.js#L151
-			'test/jshandle.spec.js',
-
-			// `package` keyword
-			// https://github.com/puppeteer/puppeteer/blob/0b1a9ceee2f05f534f0d50079ece172d627a93c7/utils/apply_next_version.js#L17
-			'utils/apply_next_version.js',
-
-			// Global return
-			'utils/fetch_devices.js',
+			// These files use `flow`
+			'**/*.js',
 		],
 	},
 	{
@@ -132,7 +171,6 @@ export default [
 			'scripts/create-package.js', // This file use `package` keyword as variable
 		],
 	},
-	'https://github.com/ReactTraining/react-router',
 	'https://github.com/mozilla/pdf.js',
 	// #912
 	{
@@ -146,37 +184,13 @@ export default [
 			'scripts/cypress.js',
 		],
 	},
-	// #902
-	{
-		repository: 'https://github.com/reakit/reakit',
-		ignore: [
-			'packages/reakit/jest.config.js', // This file use `package` keyword as variable
-		],
-	},
 	// #903
 	'https://github.com/mattermost/mattermost-webapp',
-	// #1030
-	'https://github.com/astrofox-io/astrofox',
-	// #1075
-	'https://github.com/jaredLunde/masonic',
 	// These two project use `decorator`, try to enable when we use `@babel/eslint-parser`
 	// 'https://github.com/untitled-labs/metabase-custom',
 	// 'https://github.com/TheThingsNetwork/lorawan-stack',
-].map(project => {
-	if (typeof project === 'string') {
-		project = {repository: project};
-	}
-
-	const {
-		repository,
-		name = repository.split('/').pop(),
-		ignore = [],
-	} = project;
-
-	return {
-		...project,
-		name,
-		repository,
-		ignore,
-	};
-});
+].flatMap((projectOrProjects, index) =>
+	Array.isArray(projectOrProjects)
+		? projectOrProjects.map(project => ({...normalizeProject(project), group: index}))
+		: [{...normalizeProject(projectOrProjects), group: index}],
+);

test/integration/readme.md

@@ -2,4 +2,4 @@
 
 To run the integration tests, go to the project root, and run `$ npm run integration`.
 
-To run tests on specific projects, run `$ npm run integration projectName1 projectName2 … projectNameN`. The project names can be found in [`projects.js`](projects.js).
+To run tests on specific projects, run `$ npm run integration projectName1 projectName2 … projectNameN`. The project names can be found in [`projects.mjs`](projects.mjs).