Skip to content

Commit 67bc21f

Browse files
flashdesignoryflashdesignory
authored
Runner Cleanup: SuiteRunner & TestRunner classes (#452)
1 parent e70778f commit 67bc21f

File tree

4 files changed

+204
-160
lines changed

4 files changed

+204
-160
lines changed

‎resources/benchmark-runner.mjs‎

Lines changed: 3 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { Metric } from "./metric.mjs";
22
import { params } from "./params.mjs";
3-
import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
3+
import { SUITE_RUNNER_LOOKUP } from "./suite-runner.mjs";
44

55
const performance = globalThis.performance;
66

@@ -223,7 +223,7 @@ function geomeanToScore(geomean) {
223223
// The WarmupSuite is used to make sure all runner helper functions and
224224
// classes are compiled, to avoid unnecessary pauses due to delayed
225225
// compilation of runner methods in the middle of the measuring cycle.
226-
const WarmupSuite = {
226+
export const WarmupSuite = {
227227
name: "Warmup",
228228
url: "warmup/index.html",
229229
async prepare(page) {
@@ -410,7 +410,7 @@ export class BenchmarkRunner {
410410
// FIXME: Encapsulate more state in the SuiteRunner.
411411
// FIXME: Return and use measured values from SuiteRunner.
412412
const suiteRunnerClass = SUITE_RUNNER_LOOKUP[suite.type ?? "default"];
413-
const suiteRunner = new suiteRunnerClass(this._measuredValues, this._frame, this._page, this._client, suite);
413+
const suiteRunner = new suiteRunnerClass(this._frame, this._page, params, suite, this._client, this._measuredValues);
414414
await suiteRunner.run();
415415
}
416416

@@ -484,151 +484,3 @@ export class BenchmarkRunner {
484484
metric.computeAggregatedMetrics();
485485
}
486486
}
487-
488-
// FIXME: Create AsyncSuiteRunner subclass.
489-
// FIXME: Create RemoteSuiteRunner subclass.
490-
export class SuiteRunner {
491-
constructor(measuredValues, frame, page, client, suite) {
492-
// FIXME: Create SuiteRunner-local measuredValues.
493-
this._suiteResults = measuredValues.tests[suite.name];
494-
if (!this._suiteResults) {
495-
this._suiteResults = { tests: {}, total: 0 };
496-
measuredValues.tests[suite.name] = this._suiteResults;
497-
}
498-
this._measuredValues = measuredValues;
499-
this._frame = frame;
500-
this._page = page;
501-
this._client = client;
502-
this._suite = suite;
503-
}
504-
505-
async run() {
506-
await this._prepareSuite();
507-
await this._runSuite();
508-
}
509-
510-
async _prepareSuite() {
511-
const suiteName = this._suite.name;
512-
const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
513-
const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
514-
515-
performance.mark(suitePrepareStartLabel);
516-
await this._loadFrame();
517-
await this._suite.prepare(this._page);
518-
performance.mark(suitePrepareEndLabel);
519-
520-
performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
521-
}
522-
523-
async _runSuite() {
524-
const suiteName = this._suite.name;
525-
const suiteStartLabel = `suite-${suiteName}-start`;
526-
const suiteEndLabel = `suite-${suiteName}-end`;
527-
528-
performance.mark(suiteStartLabel);
529-
for (const test of this._suite.tests)
530-
await this._runTestAndRecordResults(test);
531-
performance.mark(suiteEndLabel);
532-
533-
performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
534-
this._validateSuiteTotal();
535-
}
536-
537-
_validateSuiteTotal() {
538-
// When the test is fast and the precision is low (for example with Firefox'
539-
// privacy.resistFingerprinting preference), it's possible that the measured
540-
// total duration for an entire is 0.
541-
const suiteTotal = this._suiteResults.total;
542-
if (suiteTotal === 0)
543-
throw new Error(`Got invalid 0-time total for suite ${this._suite.name}: ${suiteTotal}`);
544-
}
545-
546-
async _loadFrame() {
547-
return new Promise((resolve, reject) => {
548-
const frame = this._page._frame;
549-
frame.onload = () => resolve();
550-
frame.onerror = () => reject();
551-
frame.src = this._suite.url;
552-
});
553-
}
554-
555-
async _runTestAndRecordResults(test) {
556-
if (this._client?.willRunTest)
557-
await this._client.willRunTest(this._suite, test);
558-
559-
// Prepare all mark labels outside the measuring loop.
560-
const suiteName = this._suite.name;
561-
const testName = test.name;
562-
const startLabel = `${suiteName}.${testName}-start`;
563-
const syncEndLabel = `${suiteName}.${testName}-sync-end`;
564-
const asyncStartLabel = `${suiteName}.${testName}-async-start`;
565-
const asyncEndLabel = `${suiteName}.${testName}-async-end`;
566-
567-
let syncTime;
568-
let asyncStartTime;
569-
let asyncTime;
570-
const runSync = () => {
571-
if (params.warmupBeforeSync) {
572-
performance.mark("warmup-start");
573-
const startTime = performance.now();
574-
// Infinite loop for the specified ms.
575-
while (performance.now() - startTime < params.warmupBeforeSync)
576-
continue;
577-
performance.mark("warmup-end");
578-
}
579-
performance.mark(startLabel);
580-
const syncStartTime = performance.now();
581-
test.run(this._page);
582-
const syncEndTime = performance.now();
583-
performance.mark(syncEndLabel);
584-
585-
syncTime = syncEndTime - syncStartTime;
586-
587-
performance.mark(asyncStartLabel);
588-
asyncStartTime = performance.now();
589-
};
590-
const measureAsync = () => {
591-
// Some browsers don't immediately update the layout for paint.
592-
// Force the layout here to ensure we're measuring the layout time.
593-
const height = this._frame.contentDocument.body.getBoundingClientRect().height;
594-
const asyncEndTime = performance.now();
595-
asyncTime = asyncEndTime - asyncStartTime;
596-
this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination.
597-
performance.mark(asyncEndLabel);
598-
if (params.warmupBeforeSync)
599-
performance.measure("warmup", "warmup-start", "warmup-end");
600-
const suiteName = this._suite.name;
601-
const testName = test.name;
602-
performance.measure(`${suiteName}.${testName}-sync`, startLabel, syncEndLabel);
603-
performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
604-
};
605-
606-
const report = () => this._recordTestResults(test, syncTime, asyncTime);
607-
const invokerClass = TEST_INVOKER_LOOKUP[params.measurementMethod];
608-
const invoker = new invokerClass(runSync, measureAsync, report, params);
609-
610-
return invoker.start();
611-
}
612-
613-
async _recordTestResults(test, syncTime, asyncTime) {
614-
// Skip reporting updates for the warmup suite.
615-
if (this._suite === WarmupSuite)
616-
return;
617-
618-
const total = syncTime + asyncTime;
619-
this._suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
620-
this._suiteResults.total += total;
621-
622-
if (this._client?.didRunTest)
623-
await this._client.didRunTest(this._suite, test);
624-
}
625-
}
626-
627-
// FIXME: implement remote steps
628-
class RemoteSuiteRunner extends SuiteRunner {}
629-
630-
const SUITE_RUNNER_LOOKUP = {
631-
__proto__: null,
632-
default: SuiteRunner,
633-
remote: RemoteSuiteRunner,
634-
};

‎resources/suite-runner.mjs‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import { TestRunner } from "./test-runner.mjs";
2+
import { WarmupSuite } from "./benchmark-runner.mjs";
3+
4+
// FIXME: Create AsyncSuiteRunner subclass.
5+
// FIXME: Create RemoteSuiteRunner subclass.
6+
export class SuiteRunner {
7+
#frame;
8+
#page;
9+
#params;
10+
#suite;
11+
#client;
12+
#suiteResults;
13+
14+
constructor(frame, page, params, suite, client, measuredValues) {
15+
// FIXME: Create SuiteRunner-local measuredValues.
16+
this.#suiteResults = measuredValues.tests[suite.name];
17+
if (!this.#suiteResults) {
18+
this.#suiteResults = { tests: {}, total: 0 };
19+
measuredValues.tests[suite.name] = this.#suiteResults;
20+
}
21+
this.#frame = frame;
22+
this.#page = page;
23+
this.#client = client;
24+
this.#suite = suite;
25+
this.#params = params;
26+
}
27+
28+
async run() {
29+
await this._prepareSuite();
30+
await this._runSuite();
31+
}
32+
33+
async _prepareSuite() {
34+
const suiteName = this.#suite.name;
35+
const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
36+
const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
37+
38+
performance.mark(suitePrepareStartLabel);
39+
await this._loadFrame();
40+
await this.#suite.prepare(this.#page);
41+
performance.mark(suitePrepareEndLabel);
42+
43+
performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
44+
}
45+
46+
async _runSuite() {
47+
const suiteName = this.#suite.name;
48+
const suiteStartLabel = `suite-${suiteName}-start`;
49+
const suiteEndLabel = `suite-${suiteName}-end`;
50+
51+
performance.mark(suiteStartLabel);
52+
for (const test of this.#suite.tests) {
53+
if (this.#client?.willRunTest)
54+
await this.#client.willRunTest(this.#suite, test);
55+
56+
const testRunner = new TestRunner(this.#frame, this.#page, this.#params, this.#suite, test, this._recordTestResults);
57+
await testRunner.runTest();
58+
}
59+
performance.mark(suiteEndLabel);
60+
61+
performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
62+
this._validateSuiteTotal();
63+
}
64+
65+
_validateSuiteTotal() {
66+
// When the test is fast and the precision is low (for example with Firefox'
67+
// privacy.resistFingerprinting preference), it's possible that the measured
68+
// total duration for an entire is 0.
69+
const suiteTotal = this.#suiteResults.total;
70+
if (suiteTotal === 0)
71+
throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`);
72+
}
73+
74+
async _loadFrame() {
75+
return new Promise((resolve, reject) => {
76+
const frame = this.#frame;
77+
frame.onload = () => resolve();
78+
frame.onerror = () => reject();
79+
frame.src = this.#suite.url;
80+
});
81+
}
82+
83+
_recordTestResults = async (test, syncTime, asyncTime) => {
84+
// Skip reporting updates for the warmup suite.
85+
if (this.#suite === WarmupSuite)
86+
return;
87+
88+
const total = syncTime + asyncTime;
89+
this.#suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
90+
this.#suiteResults.total += total;
91+
92+
if (this.#client?.didRunTest)
93+
await this.#client.didRunTest(this.#suite, test);
94+
};
95+
}
96+
97+
// FIXME: implement remote steps
98+
class RemoteSuiteRunner extends SuiteRunner {}
99+
100+
export const SUITE_RUNNER_LOOKUP = {
101+
__proto__: null,
102+
default: SuiteRunner,
103+
remote: RemoteSuiteRunner,
104+
};

‎resources/test-runner.mjs‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
2+
3+
export class TestRunner {
4+
#frame;
5+
#page;
6+
#params;
7+
#suite;
8+
#test;
9+
#callback;
10+
11+
constructor(frame, page, params, suite, test, callback) {
12+
this.#suite = suite;
13+
this.#test = test;
14+
this.#params = params;
15+
this.#callback = callback;
16+
17+
this.#page = page;
18+
this.#frame = frame;
19+
}
20+
21+
async runTest() {
22+
// Prepare all mark labels outside the measuring loop.
23+
const suiteName = this.#suite.name;
24+
const testName = this.#test.name;
25+
const syncStartLabel = `${suiteName}.${testName}-start`;
26+
const syncEndLabel = `${suiteName}.${testName}-sync-end`;
27+
const asyncStartLabel = `${suiteName}.${testName}-async-start`;
28+
const asyncEndLabel = `${suiteName}.${testName}-async-end`;
29+
30+
let syncTime;
31+
let asyncStartTime;
32+
let asyncTime;
33+
34+
const runSync = () => {
35+
if (this.#params.warmupBeforeSync) {
36+
performance.mark("warmup-start");
37+
const startTime = performance.now();
38+
// Infinite loop for the specified ms.
39+
while (performance.now() - startTime < this.#params.warmupBeforeSync)
40+
continue;
41+
performance.mark("warmup-end");
42+
}
43+
performance.mark(syncStartLabel);
44+
const syncStartTime = performance.now();
45+
this.#test.run(this.#page);
46+
const syncEndTime = performance.now();
47+
performance.mark(syncEndLabel);
48+
49+
syncTime = syncEndTime - syncStartTime;
50+
51+
performance.mark(asyncStartLabel);
52+
asyncStartTime = performance.now();
53+
};
54+
const measureAsync = () => {
55+
const bodyReference = this.#frame ? this.#frame.contentDocument.body : document.body;
56+
const windowReference = this.#frame ? this.#frame.contentWindow : window;
57+
// Some browsers don't immediately update the layout for paint.
58+
// Force the layout here to ensure we're measuring the layout time.
59+
const height = bodyReference.getBoundingClientRect().height;
60+
windowReference._unusedHeightValue = height; // Prevent dead code elimination.
61+
62+
const asyncEndTime = performance.now();
63+
performance.mark(asyncEndLabel);
64+
65+
asyncTime = asyncEndTime - asyncStartTime;
66+
67+
if (this.#params.warmupBeforeSync)
68+
performance.measure("warmup", "warmup-start", "warmup-end");
69+
performance.measure(`${suiteName}.${testName}-sync`, syncStartLabel, syncEndLabel);
70+
performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
71+
};
72+
73+
const report = () => this.#callback(this.#test, syncTime, asyncTime);
74+
const invokerClass = TEST_INVOKER_LOOKUP[this.#params.measurementMethod];
75+
const invoker = new invokerClass(runSync, measureAsync, report, this.#params);
76+
77+
return invoker.start();
78+
}
79+
}

0 commit comments

Comments
 (0)