✨ add assertions, test runners, improve output

Author: ctcpip

security/site/cve-data.mjs

@@ -1,12 +1,41 @@
+/**
+ * map of all CVEs affecting jQuery
+ * versions === affected versions per the CVE
+ * exceptions === affected versions per the CVE that we cannot reproduce
+ */
 const cveMap = new Map([
-  ['2011-4969', { versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2'] }],
-  ['2012-6708', { versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3'] }],
-	['2015-9251', { versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3', '1.12.4', '2.2.4'] }],
-	['2019-11358', { versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3', '1.12.4', '2.2.4'] }],
-	['2020-7656', { versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3'] }],
-	['2020-11022', { versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3', '1.12.4', '2.2.4'] }],
-	['2020-11023', { versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3', '1.12.4', '2.2.4'] }],
-	['2020-23064', { versions: ['2.2.4'] }],
+  ['2011-4969', {
+		versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2'],
+		exceptions: [],
+	}],
+  ['2012-6708', {
+		versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3'],
+		exceptions: [],
+	}],
+	['2015-9251', {
+		versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3', '1.12.4', '2.2.4'],
+		exceptions: ['1.2.6', '1.3.2'],
+	}],
+	['2019-11358', {
+		versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3', '1.12.4', '2.2.4'],
+		exceptions: [],
+	}],
+	['2020-7656', {
+		versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3'],
+		exceptions: [],
+	}],
+	['2020-11022', {
+		versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3', '1.12.4', '2.2.4'],
+		exceptions: [],
+	}],
+	['2020-11023', {
+		versions: ['1.2.6', '1.3.2', '1.4.4', '1.5.2', '1.6.4', '1.7.2', '1.8.3', '1.12.4', '2.2.4'],
+		exceptions: [],
+	}],
+	['2020-23064', {
+		versions: ['2.2.4'],
+		exceptions: [],
+	}],
 ]);
 
 const jQueryVersions = new Set(Array.from(cveMap, ([name, value]) => (value.versions)).flat());

security/test/package.json

@@ -1,6 +1,11 @@
 {
   "private": true,
   "scripts": {
+    "//": "use node test runner",
+    "node": "npm test node",
+    "//": "use tap test runner",
+    "tap": "npm test tap",
+    "//": "run directly; raw tap output",
     "test": "./test.sh"
   },
   "dependencies": {

security/test/test.mjs

@@ -5,18 +5,23 @@ import { cveMap, getPatchedVersion, jQueryVersions } from '../site/cve-data.mjs'
 import t from 'tap'
 import chalk from 'chalk';
 
-function banner(txt) {
-	console.log(chalk.magenta(`
+// these are fully patched
+const patchedVersions = [
+	'1.2.7-sec',
+	'1.6.5-sec'
+];
+
+function banner(txt, {borderColor = 'magenta', textColor = 'cyan'} = {borderColor: 'magenta',  textColor: 'cyan'}) {
+	console.log(chalk[borderColor](`
 --------------------------------------------------------------------------------
-  ${chalk.cyan(txt)}
+  ${chalk[textColor](txt)}
 --------------------------------------------------------------------------------
 `))
 }
 
-banner('running jQuery security tests...');
-
 const platform = os.platform();
-console.log(chalk.white(`  platform detected: ${platform}`));
+banner(`platform detected: ${platform}`, { borderColor:'white', textColor: 'white' });
+banner('running jQuery security tests...');
 
 const baseURL = 'http://127.0.0.1:3333/index.html';
 const timeout = 5 * 1000;
@@ -34,11 +39,11 @@ else {
 cmd += `--headless=old --virtual-time-budget=${timeout} --run-all-compositor-stages-before-draw --dump-dom `;
 
 for (const v of jQueryVersions) {
-	await t.test(v, async => testJQuery(v));
-	await t.test(getPatchedVersion(v), async => testJQuery(v, true));
+	await t.test(`validate jQuery v${v}`, async t => testJQuery(v, false, t));
+	await t.test(`validate jQuery v${getPatchedVersion(v)}`, async t => testJQuery(v, true, t));
 }
 
-async function testJQuery(version, patched) {
+async function testJQuery(version, patched, t) {
 
 	const effectiveVersion = patched ? getPatchedVersion(version) : version;
 
@@ -56,17 +61,37 @@ async function testJQuery(version, patched) {
 		if(cve[1].versions.includes(version)) {
 			const cveName = `CVE-${cve[0]}`
 			const status = d.querySelector(`#${cveName} .cve__footer-status`).textContent;
+			const notReproducible = status.startsWith(`Can't`);
 
-			if(status.startsWith(`Can't`)) {
+			if(notReproducible) {
 				console.log(chalk.green(`${cveName.padEnd(14)}  -  ${status}`));
 			}
 			else {
 				console.log(chalk.red(status
 					.replace('CVE', `${cveName.padEnd(14)}  - `)
 				));
 			}
+
+			if(patched && patchedVersions.includes(effectiveVersion)) {
+				t.ok(notReproducible, `${cveName} should be patched in v${effectiveVersion}`);
+			}
+			else {
+				if(cve[1].exceptions.includes(version)) {
+					t.ok(notReproducible, `${cveName} is supposed be reproducible in v${effectiveVersion} according to the CVE but it can't be reproduced`);
+				}
+				else {
+					t.notOk(notReproducible, `${cveName} should be reproducible in v${effectiveVersion}`);
+				}
+			}
 		}
 	}
 }
 
-banner('...done');
+banner(`...done`);
+
+if(t.counts.fail) {
+	banner('FAIL 💔', { borderColor: 'red', textColor: 'red'});
+}
+else {
+	banner('PASS 💚', { borderColor: 'green', textColor: 'green'});
+}

security/test/test.sh

@@ -1,7 +1,15 @@
 #!/bin/sh
 
 node ../server/index.cjs & SERVER_PID=$! ;
-tap run --allow-empty-coverage;
+
+if [ "$1" = "node" ] ; then
+	node --test ;
+elif [ "$1" = "tap" ] ; then
+	tap run --allow-empty-coverage;
+else
+	node test.mjs;
+fi
+
 TEST_EXIT=$? ;
 kill $SERVER_PID ;
 exit $TEST_EXIT ;