Add support for reporters (#129)

* add support for external reporters * reporters tests * add reporters documentation * fix reporters merging from CLI * reporters: built-in report map object and results method refactor * Update lib/helpers/resolver.js Co-authored-by: Joey Ciechanowicz <1148637+joeyciechanowicz@users.noreply.github.com> * documentation fix * refactor fallback values * Update lib/helpers/reporter.js Co-authored-by: Aaron Goldenthal <aarongoldenthal@users.noreply.github.com> * Update lib/pa11y-ci.js Co-authored-by: Aaron Goldenthal <aarongoldenthal@users.noreply.github.com> * move logs to a dedicated reporter * deduplicate and refactor reporters loader * review repoters methods * update reporters docs * typo * Add JSON reporter and unit tests * Move test data to mock folder * Update reporters to accept CLI/JSON reporter shorthand name * add multiple reporters tests * json reporter: resolve relative paths and ensure parent folders exist * reporters integratino test * docs: json reporter filename resolve strategy * rename reporters tests folder * Update README.md Co-authored-by: Aaron Goldenthal <aarongoldenthal@users.noreply.github.com> * Update lib/reporters/json.js Co-authored-by: Aaron Goldenthal <aarongoldenthal@users.noreply.github.com> * Fixes defaults to original values, and linting * Uses unlinkSync for v12 support Co-authored-by: Joey Ciechanowicz <1148637+joeyciechanowicz@users.noreply.github.com> Co-authored-by: Aaron Goldenthal <aarongoldenthal@users.noreply.github.com> Co-authored-by: Aaron Goldenthal <aaron.goldenthal@gmail.com> Co-authored-by: Joey Ciechanowicz <joeyciechanowicz@users.noreply.github.com>
pa11y · Jul 19, 2021 · 5c842cf · 5c842cf
1 parent 40ba01a
commit 5c842cf
Show file tree

Hide file tree

Showing 23 changed files with 5,663 additions and 112 deletions.
diff --git a/README.md b/README.md
@@ -21,13 +21,17 @@ Pa11y CI runs accessibility tests against multiple URLs and reports on any issue
   - [Default configuration](#default-configuration)
   - [URL configuration](#url-configuration)
   - [Sitemaps](#sitemaps)
+  - [Reporters](#reporters)
+    - [Use Multiple reporters](#use-multiple-reporters)
+    - [Write a custom reporter](#write-a-custom-reporter)
   - [Docker](#docker)
 - [Tutorials and articles](#tutorials-and-articles)
 - [Contributing](#contributing)
 - [Support and Migration](#support-and-migration)
 - [Licence](#licence)
 
 
+
 ## Requirements
 
 This command line tool requires [Node.js] 8+. You can install through npm:
@@ -55,6 +59,7 @@ Options:
   -x, --sitemap-exclude <pattern>  a pattern to find in sitemaps and exclude any url that matches
   -j, --json                       Output results as JSON
   -T, --threshold <number>         permit this number of errors, warnings, or notices, otherwise fail with exit code 2
+  --reporter <reporter>            The reporter to use. Can be "cli", "json", an npm module, or a path to a local file.
 ```
 
 ### Configuration
@@ -139,6 +144,186 @@ The above would ensure that you run Pa11y CI against local URLs instead of the l
 
 If there are items in the sitemap that you'd like to exclude from the testing (for example PDFs) you can do so using the `--sitemap-exclude` flag.
 
+## Reporters
+
+Pa11y CI includes both a CLI reporter that outputs pa11y results to the console and a JSON reporter that outputs JSON-formatted results (to the console or a file). If no reporter is specified, the CLI reporter is selected by default.  You can use the `--reporter` option to define a single reporter. The option value can be:
+- `cli` for the included CLI reporter or `json` for the included JSON reporter
+- the path of a locally installed npm module (ie: `pa11y-reporter-html`)
+- the path to a local node module relative to the current working directory (ie: `./reporters/my-reporter.js`)
+- an absolute path to a node module (ie: `/root/user/me/reporters/my-reporter.js`)
+
+Example:
+
+```
+npm install pa11y-reporter-html --save
+pa11y-ci --reporter=pa11y-reporter-html http://pa11y.org/
+```
+
+**Note**: If custom reporter(s) are specified, the default CLI reporter will be overridden.
+
+### Use Multiple reporters
+
+You can use multiple reporters by setting them on the `defaults.reporters` array in your config.  The shorthand `cli` and `json` can be included to select the included reporters.
+
+```json
+{
+    "defaults": {
+        "reporters": [
+            "cli", // <-- this is the default reporter
+            "pa11y-reporter-html",
+            "./my-local-reporter.js"
+        ]
+    },
+    "urls": [
+        "http://pa11y.org/",
+        {
+            "url": "http://pa11y.org/contributing",
+            "timeout": 50000,
+            "screenCapture": "myDir/my-screen-capture.png"
+        }
+    ]
+}
+```
+
+**Note**: If the CLI `--reporter` option is specified, it will override any reporters specified in the config file.
+
+### Reporter options
+
+Reporters can be configured, when supported, by settings the reporter as an array with its options as the second item:
+
+```json
+{
+    "defaults": {
+        "reporters": [
+            "pa11y-reporter-html",
+            ["./my-local-reporter.js", { "option1": true }] // <-- note that this is an array
+        ]
+    },
+    "urls": [
+        "http://pa11y.org/",
+        {
+            "url": "http://pa11y.org/contributing",
+            "timeout": 50000,
+            "screenCapture": "myDir/my-screen-capture.png"
+        }
+    ]
+}
+```
+
+The included CLI reporter does not support any options.
+
+The included JSON reporter outputs the results to the console by default.  It can also accept a `fileName` with a relative or absolute file name where the JSON results will be written. Relative file name will be resolved from the current working directory.
+
+```json
+{
+    "defaults": {
+        "reporters": [
+            ["json", { "fileName": "./results.json" }] // <-- note that this is an array
+        ]
+    },
+    "urls": [
+        "http://pa11y.org/"
+    ]
+}
+```
+
+### Write a custom reporter
+
+Pa11y CI reporters use an interface similar to [pa11y reporters] and support the following optional methods:
+
+- `beforeAll(urls)`: called at the beginning of the process. `urls` is the URLs array defined in your config
+- `afterAll(report)` called at the very end of the process with the following arguments:
+  - `report`: pa11y-ci report object
+  - `config`: pa11y-ci configuration object
+- `begin(url)`: called before processing each URL. `url` is the URL being processed
+-  `results(results, config)` called after pa11y test run with the following arguments:
+    - `results`: pa11y results object [URL configuration object](#url-configuration)
+    - `config`: the current [URL configuration object](#url-configuration)
+- `error(error, url, config)`: called when a test run fails with the following arguments:
+    - `error`: pa11y error message
+    - `url`: the URL being processed
+    - `config`: the current [URL configuration object](#url-configuration)
+
+Here is an example of a custom reporter writing pa11y-ci report and errors to files:
+
+```js
+const fs = require('fs');
+const { createHash } = require('crypto');
+
+// create a unique filename from URL
+function fileName(url: any, prefix = '') {
+    const hash = createHash('md5').update(url).digest('hex');
+    return `${prefix}${hash}.json`;
+}
+
+exports.afterAll = function (report) {
+    return fs.promises.writeFile('report.json', JSON.stringify(report), 'utf8');
+}
+// write error details to an individual log for each URL
+exports.error = function (error, url) {
+    const data = JSON.stringify({url, error});
+    return fs.promises.writeFile(fileName(url, 'error-'), data, 'utf8');
+}
+```
+
+#### Configurable reporters
+
+A configurable reporter is a special kind of pa11y-ci reporter exporting a single factory function as its default export.
+
+When initialized, the function receives the user configured options (if any) and pa11y-ci configuration object as argument.
+
+For example, here is a reporter writing all results to a single configurable file:
+
+```js
+// ./my-reporter.js 
+
+const fs = require('fs');
+
+module.exports = function (options) {
+    // initialize an empty report data
+    const customReport = {
+        results: {},
+        errors: [],
+        violations: 0,
+    }
+
+    const fileName = options.fileName
+
+    return {
+        // add test results to the report
+        results(results) {
+            customReport.results[results.pageUrl] = results;
+            customReport.violations += results.issues.length;
+        },
+
+        // also store errors
+        error(error, url) {
+            customReport.errors.push({ error, url });
+        },
+
+        // write to a file
+        afterAll() {
+            const data = JSON.stringify(customReport);
+            return fs.promises.writeFile(fileName, data, 'utf8');
+        }
+    }
+};
+```
+
+```json
+// configuration file
+{
+    "defaults": {
+        "reporters": [
+            ["./my-reporter.js", { "fileName": "./my-report.json" }]
+        ]
+    },
+    "urls": [
+        ...
+    ]
+}
+```
+
 ### Docker
 
 If you want to run `pa11y-ci` in a Docker container then you can use the [`buildkite/puppeteer`](https://github.com/buildkite/docker-puppeteer) image as this installs Chrome and all the required libs to run headless chrome on Linux.
@@ -171,7 +356,6 @@ ADD config.json /usr/config.json
 ENTRYPOINT ["pa11y-ci", "-c", "/usr/config.json"]
 ```
 
-
 ## Tutorials and articles
 
 Here are some useful articles written by Pa11y users and contributors:
@@ -227,6 +411,7 @@ Copyright &copy; 2016–2017, Team Pa11y
 [node.js]: https://nodejs.org/
 [pa11y]: https://github.com/pa11y/pa11y
 [pa11y configurations]: https://github.com/pa11y/pa11y#configuration
+[pa11y reporters]: https://github.com/pa11y/pa11y#reporters
 [sidekick-proposal]: https://github.com/pa11y/sidekick/blob/master/PROPOSAL.md
 [twitter]: https://twitter.com/pa11yorg
 

diff --git a/bin/pa11y-ci.js b/bin/pa11y-ci.js
@@ -51,6 +51,9 @@ commander
 		'-T, --threshold <number>',
 		'permit this number of errors, warnings, or notices, otherwise fail with exit code 2',
 		'0'
+	).option(
+		'--reporter <reporter>',
+		'the reporter to use. Can be a npm module or a path to a local file.'
 	)
 	.parse(process.argv);
 
@@ -74,6 +77,10 @@ Promise.resolve()
 		return config;
 	})
 	.then(config => {
+		// Override config reporters with CLI argument
+		if (commander.reporter) {
+			config.defaults.reporters = [commander.reporter];
+		}
 		// Actually run Pa11y CI
 		return pa11yCi(urls.concat(config.urls || []), config.defaults);
 	})

diff --git a/lib/helpers/defaults.js b/lib/helpers/defaults.js
@@ -0,0 +1,14 @@
+'use strict';
+
+const noop = () => undefined;
+
+module.exports = {
+	concurrency: 1,
+	log: {
+		error: noop,
+		info: noop
+	},
+	wrapWidth: 80,
+	reporters: ['cli'],
+	useIncognitoBrowserContext: true
+};
diff --git a/lib/helpers/loader.js b/lib/helpers/loader.js
@@ -0,0 +1,17 @@
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+
+module.exports = function loadReporter(reporter) {
+	try {
+		return require(reporter);
+	} catch (_) {
+		const localModule = path.resolve(process.cwd(), reporter);
+		if (!fs.existsSync(localModule)) {
+			console.error(`Unable to load reporter "${reporter}"`);
+			return undefined;
+		}
+		return require(localModule);
+	}
+};
diff --git a/lib/helpers/resolver.js b/lib/helpers/resolver.js
@@ -0,0 +1,32 @@
+'use strict';
+
+const loadReporter = require('./loader');
+
+const reporterShorthand = {
+	cli: require.resolve('../reporters/cli.js'),
+	json: require.resolve('../reporters/json.js')
+};
+
+module.exports = function resolveReporters(config = {}) {
+	if (!Array.isArray(config.reporters) || config.reporters.length === 0) {
+		return [];
+	}
+	return config.reporters.map(reporter => {
+		let reporterOptions = {};
+		if (Array.isArray(reporter)) {
+			[reporter, reporterOptions = {}] = reporter;
+		}
+		if (typeof reporter !== 'string') {
+			return undefined;
+		}
+		if (Object.keys(reporterShorthand).includes(reporter)) {
+			reporter = reporterShorthand[reporter];
+		}
+		const reporterModule = loadReporter(reporter);
+
+		if (typeof reporterModule === 'function') {
+			return reporterModule(reporterOptions, config);
+		}
+		return reporterModule;
+	}).filter(Boolean);
+};