Skip to content

Commit a3ed0ba

Browse files
timmywiltimmywil
committed
Tests: share queue/browser handling for all worker types
- one queue to rule them all: browserstack and selenium - retries and hard retries are now supported in selenium - selenium tests now re-use browsers in the same way as browserstack Close gh-508
1 parent f91381f commit a3ed0ba

12 files changed

+233
-269
lines changed

‎test/runner/browserstack/browsers.js renamed to ‎test/runner/browsers.js

+140-109
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,25 @@
11
import chalk from "chalk";
2-
import { getBrowserString } from "../lib/getBrowserString.js";
3-
import { createWorker, deleteWorker, getAvailableSessions } from "./api.js";
2+
import { getBrowserString } from "./lib/getBrowserString.js";
3+
import {
4+
createWorker,
5+
deleteWorker,
6+
getAvailableSessions
7+
} from "./browserstack/api.js";
8+
import createDriver from "./selenium/createDriver.js";
49

510
const workers = Object.create( null );
611

712
/**
813
* Keys are browser strings
914
* Structure of a worker:
1015
* {
11-
* debug: boolean, // Stops the worker from being cleaned up when finished
12-
* id: string,
13-
* lastTouch: number, // The last time a request was received
14-
* url: string,
15-
* browser: object, // The browser object
16+
* browser: object // The browser object
17+
* debug: boolean // Stops the worker from being cleaned up when finished
18+
* lastTouch: number // The last time a request was received
19+
* restarts: number // The number of times the worker has been restarted
1620
* options: object // The options to create the worker
21+
* url: string // The URL the worker is on
22+
* quit: function // A function to stop the worker
1723
* }
1824
*/
1925

@@ -31,70 +37,8 @@ const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
3137

3238
const WORKER_WAIT_TIME = 30000;
3339

34-
export function touchBrowser( browser ) {
35-
const fullBrowser = getBrowserString( browser );
36-
const worker = workers[ fullBrowser ];
37-
if ( worker ) {
38-
worker.lastTouch = Date.now();
39-
}
40-
}
41-
42-
async function waitForAck( worker, { fullBrowser, verbose } ) {
43-
delete worker.lastTouch;
44-
return new Promise( ( resolve, reject ) => {
45-
const interval = setInterval( () => {
46-
if ( worker.lastTouch ) {
47-
if ( verbose ) {
48-
console.log( `\n${ fullBrowser } acknowledged.` );
49-
}
50-
clearTimeout( timeout );
51-
clearInterval( interval );
52-
resolve();
53-
}
54-
}, ACKNOWLEDGE_INTERVAL );
55-
56-
const timeout = setTimeout( () => {
57-
clearInterval( interval );
58-
reject(
59-
new Error(
60-
`${ fullBrowser } not acknowledged after ${
61-
ACKNOWLEDGE_TIMEOUT / 1000 / 60
62-
}min.`
63-
)
64-
);
65-
}, ACKNOWLEDGE_TIMEOUT );
66-
} );
67-
}
68-
69-
async function restartWorker( worker ) {
70-
await cleanupWorker( worker, worker.options );
71-
await createBrowserWorker(
72-
worker.url,
73-
worker.browser,
74-
worker.options,
75-
worker.restarts + 1
76-
);
77-
}
78-
79-
export async function restartBrowser( browser ) {
80-
const fullBrowser = getBrowserString( browser );
81-
const worker = workers[ fullBrowser ];
82-
if ( worker ) {
83-
await restartWorker( worker );
84-
}
85-
}
86-
87-
async function ensureAcknowledged( worker ) {
88-
const fullBrowser = getBrowserString( worker.browser );
89-
const verbose = worker.options.verbose;
90-
try {
91-
await waitForAck( worker, { fullBrowser, verbose } );
92-
return worker;
93-
} catch ( error ) {
94-
console.error( error.message );
95-
await restartWorker( worker );
96-
}
97-
}
40+
// Limit concurrency to 8 by default in selenium
41+
const MAX_SELENIUM_CONCURRENCY = 8;
9842

9943
export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
10044
if ( restarts > MAX_WORKER_RESTARTS ) {
@@ -104,37 +48,51 @@ export async function createBrowserWorker( url, browser, options, restarts = 0 )
10448
) }`
10549
);
10650
}
107-
const verbose = options.verbose;
108-
while ( ( await getAvailableSessions() ) <= 0 ) {
51+
const { browserstack, debug, headless, runId, tunnelId, verbose } = options;
52+
while ( await maxWorkersReached( options ) ) {
10953
if ( verbose ) {
11054
console.log( "\nWaiting for available sessions..." );
11155
}
11256
await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
11357
}
11458

115-
const { debug, runId, tunnelId } = options;
11659
const fullBrowser = getBrowserString( browser );
11760

118-
const worker = await createWorker( {
119-
...browser,
120-
url: encodeURI( url ),
121-
project: "jquery-migrate",
122-
build: `Run ${ runId }`,
123-
124-
// This is the maximum timeout allowed
125-
// by BrowserStack. We do this because
126-
// we control the timeout in the runner.
127-
// See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
128-
timeout: 1800,
129-
130-
// Not documented in the API docs,
131-
// but required to make local testing work.
132-
// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
133-
"browserstack.local": true,
134-
"browserstack.localIdentifier": tunnelId
135-
} );
61+
let worker;
62+
63+
if ( browserstack ) {
64+
worker = await createWorker( {
65+
...browser,
66+
url: encodeURI( url ),
67+
project: "jquery",
68+
build: `Run ${ runId }`,
69+
70+
// This is the maximum timeout allowed
71+
// by BrowserStack. We do this because
72+
// we control the timeout in the runner.
73+
// See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
74+
timeout: 1800,
75+
76+
// Not documented in the API docs,
77+
// but required to make local testing work.
78+
// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
79+
"browserstack.local": true,
80+
"browserstack.localIdentifier": tunnelId
81+
} );
82+
worker.quit = () => deleteWorker( worker.id );
83+
} else {
84+
const driver = await createDriver( {
85+
browserName: browser.browser,
86+
headless,
87+
url,
88+
verbose
89+
} );
90+
worker = {
91+
quit: () => driver.quit()
92+
};
93+
}
13694

137-
browser.debug = !!debug;
95+
worker.debug = !!debug;
13896
worker.url = url;
13997
worker.browser = browser;
14098
worker.restarts = restarts;
@@ -147,6 +105,14 @@ export async function createBrowserWorker( url, browser, options, restarts = 0 )
147105
return ensureAcknowledged( worker );
148106
}
149107

108+
export function touchBrowser( browser ) {
109+
const fullBrowser = getBrowserString( browser );
110+
const worker = workers[ fullBrowser ];
111+
if ( worker ) {
112+
worker.lastTouch = Date.now();
113+
}
114+
}
115+
150116
export async function setBrowserWorkerUrl( browser, url ) {
151117
const fullBrowser = getBrowserString( browser );
152118
const worker = workers[ fullBrowser ];
@@ -155,6 +121,14 @@ export async function setBrowserWorkerUrl( browser, url ) {
155121
}
156122
}
157123

124+
export async function restartBrowser( browser ) {
125+
const fullBrowser = getBrowserString( browser );
126+
const worker = workers[ fullBrowser ];
127+
if ( worker ) {
128+
await restartWorker( worker );
129+
}
130+
}
131+
158132
/**
159133
* Checks that all browsers have received
160134
* a response in the given amount of time.
@@ -176,27 +150,12 @@ export async function checkLastTouches() {
176150
}
177151
}
178152

179-
export async function cleanupWorker( worker, { verbose } ) {
180-
for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
181-
if ( w === worker ) {
182-
delete workers[ fullBrowser ];
183-
await deleteWorker( worker.id );
184-
if ( verbose ) {
185-
console.log( `\nStopped ${ fullBrowser }.` );
186-
}
187-
return;
188-
}
189-
}
190-
}
191-
192153
export async function cleanupAllBrowsers( { verbose } ) {
193154
const workersRemaining = Object.values( workers );
194155
const numRemaining = workersRemaining.length;
195156
if ( numRemaining ) {
196157
try {
197-
await Promise.all(
198-
workersRemaining.map( ( worker ) => deleteWorker( worker.id ) )
199-
);
158+
await Promise.all( workersRemaining.map( ( worker ) => worker.quit() ) );
200159
if ( verbose ) {
201160
console.log(
202161
`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
@@ -209,3 +168,75 @@ export async function cleanupAllBrowsers( { verbose } ) {
209168
}
210169
}
211170
}
171+
172+
async function maxWorkersReached( {
173+
browserstack,
174+
concurrency = MAX_SELENIUM_CONCURRENCY
175+
} ) {
176+
if ( browserstack ) {
177+
return ( await getAvailableSessions() ) <= 0;
178+
}
179+
return workers.length >= concurrency;
180+
}
181+
182+
async function waitForAck( worker, { fullBrowser, verbose } ) {
183+
delete worker.lastTouch;
184+
return new Promise( ( resolve, reject ) => {
185+
const interval = setInterval( () => {
186+
if ( worker.lastTouch ) {
187+
if ( verbose ) {
188+
console.log( `\n${ fullBrowser } acknowledged.` );
189+
}
190+
clearTimeout( timeout );
191+
clearInterval( interval );
192+
resolve();
193+
}
194+
}, ACKNOWLEDGE_INTERVAL );
195+
196+
const timeout = setTimeout( () => {
197+
clearInterval( interval );
198+
reject(
199+
new Error(
200+
`${ fullBrowser } not acknowledged after ${
201+
ACKNOWLEDGE_TIMEOUT / 1000 / 60
202+
}min.`
203+
)
204+
);
205+
}, ACKNOWLEDGE_TIMEOUT );
206+
} );
207+
}
208+
209+
async function ensureAcknowledged( worker ) {
210+
const fullBrowser = getBrowserString( worker.browser );
211+
const verbose = worker.options.verbose;
212+
try {
213+
await waitForAck( worker, { fullBrowser, verbose } );
214+
return worker;
215+
} catch ( error ) {
216+
console.error( error.message );
217+
await restartWorker( worker );
218+
}
219+
}
220+
221+
async function cleanupWorker( worker, { verbose } ) {
222+
for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
223+
if ( w === worker ) {
224+
delete workers[ fullBrowser ];
225+
await worker.quit();
226+
if ( verbose ) {
227+
console.log( `\nStopped ${ fullBrowser }.` );
228+
}
229+
return;
230+
}
231+
}
232+
}
233+
234+
async function restartWorker( worker ) {
235+
await cleanupWorker( worker, worker.options );
236+
await createBrowserWorker(
237+
worker.url,
238+
worker.browser,
239+
worker.options,
240+
worker.restarts + 1
241+
);
242+
}

‎test/runner/command.js

+18-20
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ import yargs from "yargs/yargs";
22
import { browsers } from "./flags/browsers.js";
33
import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js";
44
import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
5-
import { modules } from "./modules.js";
5+
import { modules } from "./flags/modules.js";
66
import { run } from "./run.js";
7-
import { jqueryMigrate } from "./jquery-migrate.js";
8-
import { jquery } from "./jquery.js";
7+
import { jqueryMigrate } from "./flags/jquery-migrate.js";
8+
import { jquery } from "./flags/jquery.js";
99

1010
const argv = yargs( process.argv.slice( 2 ) )
1111
.version( false )
@@ -68,15 +68,23 @@ const argv = yargs( process.argv.slice( 2 ) )
6868
"Leave the browser open for debugging. Cannot be used with --headless.",
6969
conflicts: [ "headless" ]
7070
} )
71+
.option( "retries", {
72+
alias: "r",
73+
type: "number",
74+
description: "Number of times to retry failed tests by refreshing the URL."
75+
} )
76+
.option( "hard-retries", {
77+
type: "number",
78+
description:
79+
"Number of times to retry failed tests by restarting the worker. " +
80+
"This is in addition to the normal retries " +
81+
"and are only used when the normal retries are exhausted."
82+
} )
7183
.option( "verbose", {
7284
alias: "v",
7385
type: "boolean",
7486
description: "Log additional information."
7587
} )
76-
.option( "run-id", {
77-
type: "string",
78-
description: "A unique identifier for this run."
79-
} )
8088
.option( "isolate", {
8189
type: "boolean",
8290
description: "Run each module by itself in the test page. This can extend testing time."
@@ -93,19 +101,9 @@ const argv = yargs( process.argv.slice( 2 ) )
93101
"Otherwise, the --browser option will be used, " +
94102
"with the latest version/device for that browser, on a matching OS."
95103
} )
96-
.option( "retries", {
97-
alias: "r",
98-
type: "number",
99-
description: "Number of times to retry failed tests in BrowserStack.",
100-
implies: [ "browserstack" ]
101-
} )
102-
.option( "hard-retries", {
103-
type: "number",
104-
description:
105-
"Number of times to retry failed tests in BrowserStack " +
106-
"by restarting the worker. This is in addition to the normal retries " +
107-
"and are only used when the normal retries are exhausted.",
108-
implies: [ "browserstack" ]
104+
.option( "run-id", {
105+
type: "string",
106+
description: "A unique identifier for the run in BrowserStack."
109107
} )
110108
.option( "list-browsers", {
111109
type: "string",

‎test/runner/jquery-migrate.js renamed to ‎test/runner/flags/jquery-migrate.js

File renamed without changes.

‎test/runner/jquery.js renamed to ‎test/runner/flags/jquery.js

File renamed without changes.

‎test/runner/modules.js renamed to ‎test/runner/flags/modules.js

File renamed without changes.

‎test/runner/listeners.js

+11
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,17 @@
5353
return nu;
5454
}
5555
}
56+
57+
// Serialize Symbols as string representations so they are
58+
// sent over the wire after being stringified.
59+
if ( typeof value === "symbol" ) {
60+
61+
// We can *describe* unique symbols, but note that their identity
62+
// (e.g., `Symbol() !== Symbol()`) is lost
63+
var ctor = Symbol.keyFor( value ) !== undefined ? "Symbol.for" : "Symbol";
64+
return ctor + "(" + JSON.stringify( value.description ) + ")";
65+
}
66+
5667
return value;
5768
}
5869
return derez( object );

0 commit comments

Comments
 (0)