fixup! add support for retries

.github/workflows/node.js.yml

@@ -56,7 +56,7 @@ jobs:
         run: npm run build
 
       - name: Test
-        run: npm run test:unit -- -h -b ${{ matrix.BROWSER }} --jquery ${{ matrix.JQUERY }}
+        run: npm run test:unit -- -h -b ${{ matrix.BROWSER }} --jquery ${{ matrix.JQUERY }} --retries 3
 
   edge:
     runs-on: windows-latest
@@ -94,7 +94,7 @@ jobs:
         run: npm run build
 
       - name: Test
-        run: npm run test:unit -- -h -b edge --jquery ${{ matrix.JQUERY }}
+        run: npm run test:unit -- -h -b edge --jquery ${{ matrix.JQUERY }} --retries 3
 
   safari:
     runs-on: macos-latest
@@ -132,7 +132,7 @@ jobs:
         run: npm run build
 
       - name: Test
-        run: npm run test:unit -- -b safari --jquery ${{ matrix.JQUERY }}
+        run: npm run test:unit -- -b safari --jquery ${{ matrix.JQUERY }} --retries 3
 
   legacy-build:
     runs-on: ubuntu-latest

tests/runner/command.js

@@ -58,6 +58,11 @@ const argv = yargs( process.argv.slice( 2 ) )
 			"Leave the browser open for debugging. Cannot be used with --headless.",
 		conflicts: [ "headless" ]
 	} )
+	.option( "retries", {
+		alias: "r",
+		type: "number",
+		description: "Number of times to retry failed tests."
+	} )
 	.option( "concurrency", {
 		alias: "c",
 		type: "number",

tests/runner/run.js

@@ -7,7 +7,7 @@ import { generateHash } from "./lib/generateHash.js";
 import { getBrowserString } from "./lib/getBrowserString.js";
 import { suites as allSuites } from "./suites.js";
 import { cleanupAllBrowsers, touchBrowser } from "./selenium/browsers.js";
-import { addRun, getNextBrowserTest, runAll } from "./selenium/queue.js";
+import { addRun, getNextBrowserTest, retryTest, runAll } from "./selenium/queue.js";
 
 const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
 
@@ -21,6 +21,7 @@ export async function run( {
 	headless,
 	jquery: jquerys = [],
 	migrate,
+	retries = 0,
 	suite: suites = [],
 	verbose
 } ) {
@@ -88,6 +89,13 @@ export async function run( {
 
 				// Handle failure
 				if ( failed ) {
+					const retry = retryTest( reportId, retries );
+
+					// Retry if retryTest returns a test
+					if ( retry ) {
+						return retry;
+					}
+
 					errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
 				}
 

tests/runner/selenium/queue.js

@@ -1,3 +1,4 @@
+import chalk from "chalk";
 import { getBrowserString } from "../lib/getBrowserString.js";
 import {
 	checkLastTouches,
@@ -34,11 +35,30 @@ export function getNextBrowserTest( reportId ) {
 	}
 }
 
+export function retryTest( reportId, maxRetries ) {
+	if ( !maxRetries ) {
+		return;
+	}
+	const test = queue.find( ( test ) => test.id === reportId );
+	if ( test ) {
+		test.retries++;
+		if ( test.retries <= maxRetries ) {
+			console.log(
+				`\nRetrying test ${ reportId } for ${ chalk.yellow( test.options.suite ) }...${
+					test.retries
+				}`
+			);
+			return test;
+		}
+	}
+}
+
 export function addRun( url, browser, options ) {
 	queue.push( {
 		browser,
 		fullBrowser: getBrowserString( browser ),
 		id: options.reportId,
+		retries: 0,
 		url,
 		options,
 		running: false