Add support for reporters

README.md

@@ -21,11 +21,15 @@ CI runs accessibility tests against multiple URLs and reports on any issues. Thi
   - [Default configuration](#default-configuration)
   - [URL configuration](#url-configuration)
   - [Sitemaps](#sitemaps)
+  - [Reporters](#reporters)
+    - [Use Multiple reporters](#use-multiple-reporters)
+    - [Write a custom reporter](#write-a-custom-reporter)
 - [Tutorials and articles](#tutorials-and-articles)
 - [Contributing](#contributing)
 - [License](#license)
 
 
+
 ## Requirements
 
 This command line tool requires [Node.js] 8+. You can install through npm:
@@ -53,6 +57,7 @@ Options:
   -x, --sitemap-exclude <pattern>  a pattern to find in sitemaps and exclude any url that matches
   -j, --json                       Output results as JSON
   -T, --threshold <number>         permit this number of errors, warnings, or notices, otherwise fail with exit code 2
+  --reporter <reporter>            The reporter to use. Can be a npm module or a path to a local file.
 ```
 
 ### Configuration
@@ -137,6 +142,99 @@ The above would ensure that you run Pa11y CI against local URLs instead of the l
 
 If there are items in the sitemap that you'd like to exclude from the testing (for example PDFs) you can do so using the `--sitemap-exclude` flag.
 
+## Reporters
+
+Pa11y CI supports Pa11y compatible reporters. You can use the `--reporter` option to define a single reporter. The option value can be:
+- the path of a locally installed npm module (ie: `pa11y-reporter-html`)
+- the path to a local node module relative to the current working directory (ie: `./reporters/my-reporter.js`)
+- an absolute path to a node module (ie: `/root/user/me/reporters/my-reporter.js`)
+
+Example:
+
+```
+npm install pa11y-reporter-html --save
+pa11y-ci --reporter=pa11y-reporter-html http://pa11y.org/
+```
+
+**Note**: When using reporters that output to stdout, all pa11y-ci execution logs will be redirected to stderr. This allows you to
+use output redirection without issues:
+
+```
+pa11y-ci --reporter=pa11y-reporter-html http://pa11y.org/ > my-report.html
+```
+
+### Use Multiple reporters
+
+You can use multiple reporters by setting them on the `defaults.reporters` array in your config.
+
+```json
+{
+    "defaults": {
+        "reporters": [
+            "pa11y-reporter-html",
+            "./my-local-reporter.js"
+        ]
+    },
+    "urls": [
+        "http://pa11y.org/",
+        {
+            "url": "http://pa11y.org/contributing",
+            "timeout": 50000,
+            "screenCapture": "myDir/my-screen-capture.png"
+        }
+    ]
+}
+```
+
+### Write a custom reporter
+
+Pa11y CI reporters use the same interface as [pa11y reporters] with some additions:
+
+- Every reporter method receives an additional `report` argument. This object is an instance of a [JavaScript Map](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map) that can be used to initialize and collect data across each tested URL.
+- You can define a `beforeAll(urls, report)` and `afterAll(urls, report)` optional methods called respectively at the beginning and at the very end of the process with the following arguments:
+  - `urls`: the URLs array defined in your config
+  - `report`: the report object
+- The `results()` method receives a third `option` argument with the following properties:
+    - `config`: the current [URL configuration object](#url-configuration)
+    - `url`: the current URL under test
+    - `urls`: the URLs array defined in your config
+
+**Note**: to prevent a reporter from logging to stdout, ensure its methods return a falsy value or a Promise resolving to a falsy value.
+
+Here is an example of a custom reporter logging to a file
+
+```js
+const fs = require('fs');
+
+// initialize an empty report data
+// "report" is a JavaScript Map instance 
+function beforeAll(_, report) {
+    report.set('data', {
+        results: {},
+      violations: 0,
+    });
+}
+
+// add test results to the report
+function results(results, report, { url }) {
+    const data = report.get('data');
+    data.results[url] = results;
+    data.violations += results.issues.length;
+}
+
+// write to a file
+function afterAll(_, report) {
+    fs.writeFileSync('./report.json', JSON.stringify(report.get('data')), 'utf8');
+    // or Node 10+ you can use:
+    // return fs.promises.writeFile('./report.json', JSON.stringify(report.get('data')), 'utf8');`
+}
+
+module.exports = {
+    beforeAll,
+    results,
+    afterAll,
+}
+```
 
 ## Tutorials and articles
 
@@ -192,6 +290,7 @@ Copyright &copy; 2016–2017, Team Pa11y
 [node.js]: https://nodejs.org/
 [pa11y]: https://github.com/pa11y/pa11y
 [pa11y configurations]: https://github.com/pa11y/pa11y#configuration
+[pa11y reporters]: https://github.com/pa11y/pa11y#reporters
 [sidekick-proposal]: https://github.com/pa11y/sidekick/blob/master/PROPOSAL.md
 [twitter]: https://twitter.com/pa11yorg
 

bin/pa11y-ci.js

@@ -51,6 +51,9 @@ commander
 		'-T, --threshold <number>',
 		'permit this number of errors, warnings, or notices, otherwise fail with exit code 2',
 		'0'
+	).option(
+		'--reporter <reporter>',
+		'the reporter to use. Can be a npm module or a path to a local file.',
 	)
 	.parse(process.argv);
 
@@ -74,6 +77,10 @@ Promise.resolve()
 		return config;
 	})
 	.then(config => {
+		// Override config reporters with CLI argument
+		if (commander.reporter) {
+			config.defaults.reporters = [commander.reporter];
+		}
 		// Actually run Pa11y CI
 		return pa11yCi(urls.concat(config.urls || []), config.defaults);
 	})

lib/helpers/reporter.js

@@ -0,0 +1,47 @@
+'use strict';
+
+/**
+ * Build a Pa11y reporter.
+ *
+ * Same as 'pa11y/lib/reporter' but reporter methods accept multiple arguments
+ * @private
+ * @param {Object} methods - The reporter methods.
+ * @returns {Promise} Returns a promise which resolves with the new reporter.
+ */
+module.exports = function buildReporter(methods) {
+	const reporter = {
+		report: new Map(),
+		supports: methods.supports,
+		beforeAll: buildReporterMethod(methods.beforeAll),
+		afterAll: buildReporterMethod(methods.afterAll),
+		begin: buildReporterMethod(methods.begin),
+		results: buildReporterMethod(methods.results),
+		log: {
+			debug: buildReporterMethod(methods.debug),
+			error: buildReporterMethod(methods.error, 'error'),
+			info: buildReporterMethod(methods.info)
+		}
+	};
+
+	return reporter;
+};
+
+/**
+ * Build a Pa11y reporter method, making it async and only outputting when
+ * actual output is returned.
+ * @private
+ * @param {Function} method - The reporter method to build.
+ * @param {String} [consoleMethod='log'] - The console method to use in reporting.
+ * @returns {Function} Returns a built async reporter method.
+ */
+function buildReporterMethod(method, consoleMethod = 'log') {
+	if (typeof method !== 'function') {
+		return () => Promise.resolve();
+	}
+	return async (...args) => {
+		const output = await method(...args);
+		if (output) {
+			console[consoleMethod](output);
+		}
+	};
+}

lib/helpers/resolver.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const buildReporter = require('./reporter');
+
+module.exports = function resolveReporters(reporters) {
+	return [].concat(reporters).map(reporter => {
+		if (typeof reporter !== 'string') {
+			return undefined;
+		}
+		try {
+			return require(reporter);
+		} catch (_) {
+			const localModule = path.isAbsolute(reporter) ?
+				reporter : path.resolve(process.cwd(), reporter);
+			if (fs.existsSync(localModule)) {
+				return require(localModule);
+			}
+			console.error(`Unable to locale reporter "${reporter}"`);
+			return undefined;
+		}
+	}).filter(Boolean).map(reporterModule => {
+		return buildReporter(reporterModule);
+	});
+};

lib/pa11y-ci.js

@@ -11,6 +11,8 @@ const pa11y = require('pa11y');
 const queue = require('async/queue');
 const wordwrap = require('wordwrap');
 const puppeteer = require('puppeteer');
+const resolveReporters = require('./helpers/resolver');
+const {Console} = require('console');
 
 // Just an empty function to use as default
 // configuration and arguments
@@ -22,19 +24,35 @@ const noop = () => {};
 // file and is the function that actually starts to do things
 module.exports = pa11yCi;
 
+
+const defaultLog = {
+	error: noop,
+	info: noop
+};
+
+function cycleReporters(reporters, method, payload, options = {}) {
+	if (!reporters.length) {
+		return Promise.resolve();
+	}
+	return Promise.all(reporters.map(reporter => {
+		if (typeof reporter[method] === 'function') {
+			return reporter[method](payload, reporter.report, options);
+		}
+		return Promise.resolve();
+	}));
+}
+
 // The default configuration object. This is extended with
 // whatever configurations the user passes in from the
 // command line
 module.exports.defaults = {
 	concurrency: 2,
-	log: {
-		error: noop,
-		info: noop
-	},
+	log: defaultLog,
 	wrapWidth: 80,
 	useIncognitoBrowserContext: false
 };
 
+
 // This function does all the setup and actually runs Pa11y
 // against the passed in URLs. It accepts options in the form
 // of an object and returns a Promise
@@ -46,12 +64,26 @@ function pa11yCi(urls, options) {
 		// Default the passed in options
 		options = defaults({}, options, module.exports.defaults);
 
+		// Resolve reporters
+		const reporters = resolveReporters(options.reporters);
+
 		// We delete options.log because we don't want it to
 		// get passed into Pa11y – we don't want super verbose
 		// logs from it
-		const log = options.log;
+		let log = options.log;
 		delete options.log;
 
+		if (reporters.length && (log === defaultLog || log === console)) {
+			// If the user didn't set a custom logger
+			// and has set reporters
+			// redirect all logs to stderr
+			// so that results can be piped on the CLI.
+			log = new Console({stdout: process.stderr,
+				stderr: process.stderr});
+		}
+
+		await cycleReporters(reporters, 'beforeAll', urls);
+
 
 		// Create a Pa11y test function and an async queue
 		const taskQueue = queue(testRunner, options.concurrency);
@@ -105,6 +137,9 @@ function pa11yCi(urls, options) {
 				url = config.url;
 				config = defaults({}, config, options);
 			}
+
+			await cycleReporters(reporters, 'begin', url);
+
 			config.browser = config.useIncognitoBrowserContext ?
 				await testBrowser.createIncognitoBrowserContext() :
 				testBrowser;
@@ -113,6 +148,9 @@ function pa11yCi(urls, options) {
 			// results to the report object
 			try {
 				const results = await pa11y(url, config);
+				await cycleReporters(reporters, 'results', results, {config,
+					url,
+					urls});
 				processResults(results, config, url);
 			} catch (error) {
 				log.error(` ${chalk.cyan('>')} ${url} - ${chalk.red('Failed to run')}`);
@@ -163,9 +201,8 @@ function pa11yCi(urls, options) {
 				});
 				log.error(chalk.red(`\n✘ ${passRatio}`));
 			}
-
 			// Resolve the promise with the report
-			resolve(report);
+			cycleReporters(reporters, 'afterAll', urls).then(() => resolve(report));
 		}
 
 	});