Merge pull request #385 from Financial-Times/obtPa11y

Added pa11y test to obt, with its own test. README updated

README.md

@@ -9,6 +9,7 @@ If you have any issues with OBT, please check out [troubleshooting guide](https:
 1. Install these dependencies:
 	* [node.js](http://nodejs.org/)
 	* [Ruby](https://www.ruby-lang.org/en/) (Macs typically ship with Ruby by default)
+	* [PhantomJS](http://phantomjs.org/)
 
 2. Install the build tools globally:
 
@@ -167,6 +168,9 @@ Config accepts:
 	- environments: `String` Comma separated list of environments from your nightwatch config file to run your tests on. (Default: `chrome_latest,chrome_latest-1,firefox_latest,firefox_latest-1,ie8_Grid,ie9_Grid,ie10_Grid,ie11_Grid,safari7_Grid`)
 	- testsPath: `String` Relative path from your project's root directory to where your nightwatch tests are. (Default: `test/browser/tests`)
 	- browserTestFailBuild: `Boolean` by default the failed browser tests will not fail the task, however you can override the default behaviour by setting this option to `true`. Default: `false`
+* __pa11yTest(gulp, config)__ PhantomJS is required for running [Pa11y](pa11y.org). The test will not fail if the pa11y.html file is not found. Config accepts:
+	- pa11yPath: `String` Relative path from your project's root directory to where your pa11y test is. (Default: `/demos/local/pa11y.html`)
+	- pa11yIgnore: `String` Types and codes of messages to ignore, a repeatable value or separated by semi-colons. (Default: `WCAG2AA.Principle2.Guideline2_4.2_4_1.G1,G123,G124.NoSuchID;warning;notice`)
 
 ## gulpfile usage
 

lib/tasks/demo.js

@@ -19,8 +19,7 @@ portfinder.basePort = 8080;
 const builtFiles = {};
 const defaultDemoConfig = {
 	documentClasses: '',
-	description: '',
-	expanded: true
+	description: ''
 };
 let server;
 
@@ -36,7 +35,7 @@ function buildSass(gulp, buildConfig) {
 		let prefixSass = '';
 		// If module has o-assets as a dependency, set local demos to use local assets
 		// to use the latest ones in the repo
-		if (bowerConfig.dependencies && bowerConfig.dependencies['o-assets']) {
+		if (bowerConfig && bowerConfig.dependencies && bowerConfig.dependencies['o-assets']) {
 			const moduleName = bowerConfig.name;
 			prefixSass = '@import \'o-assets/main\';\n' +
 				'@include oAssetsSetModulePaths((' + moduleName + ': ""));\n';

lib/tasks/test.js

@@ -5,6 +5,7 @@ const path = require('path');
 const commandLine = require('../helpers/command-line');
 const files = require('../helpers/files');
 const log = require('../helpers/log');
+const url = require('url');
 
 function silentCompilationTest(gulp, silent) {
 	const sass = require('./build').sass;
@@ -61,9 +62,10 @@ function runSilentModeTest(gulp) {
 
 module.exports = function(gulp, config) {
 	return new Promise(function(resolve, reject) {
-		return Promise.all([
+		Promise.all([
 			module.exports.npmTest(),
-			runSilentModeTest(gulp)
+			runSilentModeTest(gulp),
+			module.exports.pa11yTest(gulp, config)
 		])
 		.then(function() {
 			if (config.browserTest) {
@@ -79,7 +81,8 @@ module.exports = function(gulp, config) {
 			} else {
 				resolve();
 			}
-		}).catch(function(error) {
+		})
+		.catch(function(error) {
 			reject(error);
 		});
 	});
@@ -107,6 +110,8 @@ module.exports.npmTest = function() {
 					log.secondary(output.stdout);
 					reject(output.stderr);
 				});
+		} else {
+			resolve();
 		}
 	});
 };
@@ -264,5 +269,46 @@ module.exports.browserTest = function(gulp, config) {
 	});
 };
 
+module.exports.pa11yTest = function(gulp, config) {
+	log.secondary('Running "pa11y test"...');
+	config = config || {};
+	const defaultSrc = path.join(process.cwd(), '/demos/local/pa11y.html');
+	// ignoring by default warnings, notices and the href="#" error
+	const defaultIgnore = ['WCAG2AA.Principle2.Guideline2_4.2_4_1.G1,G123,G124.NoSuchID', 'warning', 'notice'];
+	const src = (config.pa11yPath !== undefined) ? url.parse(path.join(process.cwd(), config.pa11yPath)) : url.parse(defaultSrc);
+	const ignore = (config.pa11yIgnore !== undefined) ? config.pa11yIgnore.split(';') : defaultIgnore;
+
+	return new Promise(function(resolve, reject) {
+		fs.readFile(src.href, function(err) {
+			if (err === null) {
+				const pa11y = require('pa11y');
+				const pa11yCliReporter = require('pa11y/reporter/cli');
+
+				const test = pa11y({
+					log: {
+						debug: console.log.bind(console),
+						error: console.error.bind(console),
+						info: console.info.bind(console)
+					},
+					ignore: ignore
+				});
+				src.href = src.protocol ? src.href : 'file://' + src.href;
+
+				test.run(src.href, function(error, result) {
+					if (error) {
+						reject(error);
+					} else {
+						pa11yCliReporter.results(result, src.href);
+						resolve(result);
+					}
+				});
+			} else {
+				log.secondary(`Could not open ${src.href}`);
+				resolve();
+			};
+		});
+	});
+};
+
 module.exports.watchable = true;
 module.exports.description = 'Test if Sass silent compilation follows the Origami specification';

package.json

@@ -48,6 +48,7 @@
     "nightwatch": "^0.8.9",
     "node.extend": "^1.1.5",
     "npmconf": "^2.1.2",
+    "pa11y":  "^4.0.0",
     "portfinder": "^0.4.0",
     "raw-loader": "^0.5.1",
     "semver": "^5.1.0",

test/fixtures/o-test/demos/src/config.json

@@ -8,14 +8,13 @@
 		{
 			"name": "test1",
 			"description": "First test",
-			"template": "demos/src/test1.mustache",
-			"expanded": true
+			"template": "demos/src/test1.mustache"
 		},
 		{
 			"name": "test2",
 			"description": "Second test",
 			"template": "demos/src/test2.mustache",
-			"expanded": false
+			"hidden": true
 		}
 	]
 }

test/fixtures/o-test/demos/src/oldconfig.json

@@ -7,13 +7,12 @@
 	"demos": {
 		"test1": {
 			"description": "First test",
-			"template": "demos/src/test1.mustache",
-			"expanded": true
+			"template": "demos/src/test1.mustache"
 		},
 		"test2": {
 			"description": "Second test",
 			"template": "demos/src/test2.mustache",
-			"expanded": false
+			"hidden": true
 		}
 	}
-}
+}

test/fixtures/o-test/demos/src/pa11y.mustache

@@ -0,0 +1,6 @@
+<!-- pa11y error, no contrast-->
+<button style="background-color:blue; color:blue">No contrast</button>
+<!-- pa11y warning, not logically nested-->
+<h2>Not logically nested</h2>
+<!-- no errors -->
+<button>Click me</button>

test/fixtures/o-test/origami.json

@@ -20,7 +20,13 @@
             "name": "test2",
             "template": "demos/src/test2.mustache",
             "path": "/demos/test2.html",
+            "hidden": true,
             "description": "Second test"
+        },
+        {
+              "name": "pa11y",
+              "template": "demos/src/pa11y.mustache",
+              "description": "Pa11y test"
         }
     ],
     "browserFeatures": {

test/tasks/test.test.js

@@ -0,0 +1,81 @@
+/* global describe, it, before, after */
+'use strict';
+
+const expect = require('expect.js');
+const gulp = require('gulp');
+
+const fs = require('fs-extra');
+const path = require('path');
+
+const demo = require('../../lib/tasks/demo');
+const test = require('../../lib/tasks/test');
+
+const obtPath = process.cwd();
+const oTestPath = 'test/fixtures/o-test';
+
+describe('Test task', function() {
+	describe('Pa11y', function() {
+		const pathSuffix = '-test-pa11y';
+		const testTestPath = path.resolve(obtPath, oTestPath + pathSuffix);
+
+		before(function(done) {
+			fs.copySync(path.resolve(obtPath, oTestPath), testTestPath);
+			process.chdir(testTestPath);
+			const demoStream = demo(gulp, {
+				demoConfig: 'origami.json',
+				demoFilter: ['pa11y']
+			});
+			demoStream.on('end', done);
+			demoStream.resume();
+		});
+
+		after(function() {
+			process.chdir(obtPath);
+			fs.removeSync(testTestPath);
+		});
+
+		it('should not fail when the file is not found', function() {
+			// run pa11y subtask
+			return test.pa11yTest(gulp, {
+					// set an invalid path
+					pa11yPath: './file.html'
+				})
+				.then(function(results) {
+					expect(results).to.be(undefined);
+				});
+		});
+
+		it('should run pa11y correctly', function(done) {
+			// run pa11y subtask
+			const res = test.pa11yTest();
+			res
+				.then(function(results) {
+					expect(results[0].type).to.be('error');
+					done();
+				})
+				.catch(function errorHandler() {
+					// it should throw an error, not resolve
+					expect(true).to.be(false);
+					done();
+				});
+		});
+
+		it('should run pa11y with custom ignore options', function(done) {
+			// run pa11y subtask
+			const res = test.pa11yTest(gulp, {
+				// set custom ignore options
+				pa11yIgnore: 'error;notice'
+			});
+			res
+				.then(function(results) {
+					expect(results[0].type).to.be('warning');
+					done();
+				})
+				.catch(function errorHandler() {
+					// it should throw an error, not resolve
+					expect(true).to.be(false);
+					done();
+				});
+		});
+	});
+});