diff --git a/.github/workflows/update-test262-tests.yml b/.github/workflows/update-test262-tests.yml new file mode 100644 index 00000000000000..9241e1de878234 --- /dev/null +++ b/.github/workflows/update-test262-tests.yml @@ -0,0 +1,58 @@ +name: Update Test262 tests + +on: + # Trigger at every Sunday UTC noon, or manually. + schedule: + - cron: '0 12 * * 0' + workflow_dispatch: + +jobs: + update-test262: + runs-on: ubuntu-24.04 + steps: + - name: Checkout WPT repo + uses: actions/checkout@v4 + with: + path: wpt + - name: Checkout Test262 repo + uses: actions/checkout@v4 + with: + repository: tc39/test262 + path: test262-spec + - name: Copy Test262 tests to WPT + run: | + LATEST_SHA=$(git -C test262-spec rev-parse HEAD) + echo "Latest remote Test262 SHA: $LATEST_SHA" + + mkdir -p wpt/third_party/test262/test/ + mkdir -p wpt/third_party/test262/harness/ + rsync -a --delete test262-spec/test/ wpt/third_party/test262/test/ + rsync -a --delete test262-spec/harness/ wpt/third_party/test262/harness/ + printf "[test262]\nsource = \"https://github.com/tc39/test262\"\nrev = \"${LATEST_SHA}\"\n" > wpt/third_party/test262/vendored.toml + - name: Commit changes + id: commit + continue-on-error: true + run: | + cd wpt + export BRANCH_NAME="$BRANCH_PREFIX-$(date +'%Y%m%d%H%M%S')" + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV + git config user.name "$GIT_AUTHOR_NAME" + git config user.email "$GIT_AUTHOR_EMAIL" + git checkout -B $BRANCH_NAME + git add third_party/test262/ + git commit -m "$COMMIT_TITLE" + env: + GIT_AUTHOR_NAME: "wpt-pr-bot" + GIT_AUTHOR_EMAIL: "wpt-pr-bot@users.noreply.github.com" + BRANCH_PREFIX: "test262-update" + COMMIT_TITLE: "Update Test262 tests" + - name: Create PR + if: ${{ steps.commit.outcome == 'success' }} + run: | + cd wpt + git push --set-upstream origin $BRANCH_NAME + gh pr create --title "$COMMIT_TITLE" --body "$PR_BODY" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMIT_TITLE: "Update Test262 tests" + PR_BODY: "Scheduled weekly update auto-generated by the '${{ github.workflow }}' workflow." diff --git a/infrastructure/metadata/infrastructure/test262/basic-failure.js.ini b/infrastructure/metadata/infrastructure/test262/basic-failure.js.ini new file mode 100644 index 00000000000000..f09895219ef187 --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/basic-failure.js.ini @@ -0,0 +1,4 @@ +[basic-failure.test262.html] + expected: OK + [Test] + expected: FAIL diff --git a/infrastructure/metadata/infrastructure/test262/basic.js.ini b/infrastructure/metadata/infrastructure/test262/basic.js.ini new file mode 100644 index 00000000000000..390c424933c2bf --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/basic.js.ini @@ -0,0 +1,4 @@ +[basic.test262.html] + expected: OK + [Test] + expected: PASS diff --git a/infrastructure/metadata/infrastructure/test262/error.js.ini b/infrastructure/metadata/infrastructure/test262/error.js.ini new file mode 100644 index 00000000000000..76852fe1949d5c --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/error.js.ini @@ -0,0 +1,4 @@ +[error.test262.html] + expected: OK + [Test] + expected: FAIL diff --git a/infrastructure/metadata/infrastructure/test262/module-error.js.ini b/infrastructure/metadata/infrastructure/test262/module-error.js.ini new file mode 100644 index 00000000000000..242ba0b2e1186a --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/module-error.js.ini @@ -0,0 +1,4 @@ +[module-error.test262-module.html] + expected: ERROR + [Test] + expected: FAIL diff --git a/infrastructure/metadata/infrastructure/test262/module-failure.js.ini b/infrastructure/metadata/infrastructure/test262/module-failure.js.ini new file mode 100644 index 00000000000000..259ff3ed8a942f --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/module-failure.js.ini @@ -0,0 +1,4 @@ +[module-failure.test262-module.html] + expected: OK + [Test] + expected: FAIL diff --git a/infrastructure/metadata/infrastructure/test262/module.js.ini b/infrastructure/metadata/infrastructure/test262/module.js.ini new file mode 100644 index 00000000000000..a615125e6d3458 --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/module.js.ini @@ -0,0 +1,4 @@ +[module.test262-module.html] + expected: OK + [Test] + expected: PASS diff --git a/infrastructure/metadata/infrastructure/test262/negative-error.js.ini b/infrastructure/metadata/infrastructure/test262/negative-error.js.ini new file mode 100644 index 00000000000000..666389bfbe032e --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/negative-error.js.ini @@ -0,0 +1,4 @@ +[negative-error.test262.html] + expected: ERROR + [Test] + expected: FAIL diff --git a/infrastructure/metadata/infrastructure/test262/negative-failure.js.ini b/infrastructure/metadata/infrastructure/test262/negative-failure.js.ini new file mode 100644 index 00000000000000..2d7ad8eb0c3497 --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/negative-failure.js.ini @@ -0,0 +1,4 @@ +[negative-failure.test262.html] + expected: OK + [Test] + expected: FAIL diff --git a/infrastructure/metadata/infrastructure/test262/negative.js.ini b/infrastructure/metadata/infrastructure/test262/negative.js.ini new file mode 100644 index 00000000000000..d2ba995c1925fc --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/negative.js.ini @@ -0,0 +1,4 @@ +[negative.test262.html] + expected: OK + [Test] + expected: PASS diff --git a/infrastructure/metadata/infrastructure/test262/strict-error.js.ini b/infrastructure/metadata/infrastructure/test262/strict-error.js.ini new file mode 100644 index 00000000000000..312ba4e05911de --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/strict-error.js.ini @@ -0,0 +1,4 @@ +[strict-error.test262.strict.html] + expected: ERROR + [Test] + expected: FAIL diff --git a/infrastructure/metadata/infrastructure/test262/strict-failure.js.ini b/infrastructure/metadata/infrastructure/test262/strict-failure.js.ini new file mode 100644 index 00000000000000..646caf613bf8ef --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/strict-failure.js.ini @@ -0,0 +1,4 @@ +[strict-failure.test262.strict.html] + expected: OK + [Test] + expected: FAIL diff --git a/infrastructure/metadata/infrastructure/test262/strict.js.ini b/infrastructure/metadata/infrastructure/test262/strict.js.ini new file mode 100644 index 00000000000000..7006bf6ea90b6e --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/strict.js.ini @@ -0,0 +1,4 @@ +[strict.test262.html] + expected: OK + [Test] + expected: PASS diff --git a/infrastructure/metadata/infrastructure/test262/unexpected-error.js.ini b/infrastructure/metadata/infrastructure/test262/unexpected-error.js.ini new file mode 100644 index 00000000000000..b62ca2ad14ba7b --- /dev/null +++ b/infrastructure/metadata/infrastructure/test262/unexpected-error.js.ini @@ -0,0 +1,4 @@ +[unexpected-error.test262.html] + expected: ERROR + [Test] + expected: FAIL diff --git a/infrastructure/test262/basic-failure.js b/infrastructure/test262/basic-failure.js new file mode 100644 index 00000000000000..613183fd4c7f63 --- /dev/null +++ b/infrastructure/test262/basic-failure.js @@ -0,0 +1,6 @@ +/*--- +description: A basic Test262 smoketest that fails +features: [Test262] +---*/ + +assert.sameValue(1, 2, "One should be two"); diff --git a/infrastructure/test262/basic.js b/infrastructure/test262/basic.js new file mode 100644 index 00000000000000..439a1220da4cc2 --- /dev/null +++ b/infrastructure/test262/basic.js @@ -0,0 +1,6 @@ +/*--- +description: A basic Test262 smoketest +features: [Test262] +---*/ + +assert.sameValue(1, 1, "One should be one"); diff --git a/infrastructure/test262/error.js b/infrastructure/test262/error.js new file mode 100644 index 00000000000000..3b0f7762aadeaa --- /dev/null +++ b/infrastructure/test262/error.js @@ -0,0 +1,5 @@ +/*--- +description: A smoketest that throws a Test262Error +---*/ + +throw new Test262Error("This is a deliberate test failure."); diff --git a/infrastructure/test262/module-error.js b/infrastructure/test262/module-error.js new file mode 100644 index 00000000000000..439371f5922085 --- /dev/null +++ b/infrastructure/test262/module-error.js @@ -0,0 +1,6 @@ +/*--- +description: A module smoketest that throws an unexpected ReferenceError. +flags: [module] +---*/ +import { a } from "./support/module-helper.js"; +foo.bar(); diff --git a/infrastructure/test262/module-failure.js b/infrastructure/test262/module-failure.js new file mode 100644 index 00000000000000..33ecad9eccfed7 --- /dev/null +++ b/infrastructure/test262/module-failure.js @@ -0,0 +1,6 @@ +/*--- +description: A module smoketest that fails an assertion. +flags: [module] +---*/ +import { a } from "./support/module-helper.js"; +assert.sameValue(a, 2, "a should be 2 in module"); diff --git a/infrastructure/test262/module.js b/infrastructure/test262/module.js new file mode 100644 index 00000000000000..8a3642fe4c4d89 --- /dev/null +++ b/infrastructure/test262/module.js @@ -0,0 +1,5 @@ +/*--- +description: A module smoketest +flags: [module] +---*/ +import { a } from "./support/module-helper.js"; diff --git a/infrastructure/test262/negative-error.js b/infrastructure/test262/negative-error.js new file mode 100644 index 00000000000000..2a784d688fe955 --- /dev/null +++ b/infrastructure/test262/negative-error.js @@ -0,0 +1,8 @@ +/*--- +description: A negative test that throws a different error than expected. +negative: + phase: runtime + type: TypeError +---*/ + +throw new RangeError(); diff --git a/infrastructure/test262/negative-failure.js b/infrastructure/test262/negative-failure.js new file mode 100644 index 00000000000000..883a8ad928ac2e --- /dev/null +++ b/infrastructure/test262/negative-failure.js @@ -0,0 +1,6 @@ +/*--- +description: A negative smoketest that is expected to fail because it does not throw an error. +negative: + phase: runtime + type: Test262Error +---*/ diff --git a/infrastructure/test262/negative.js b/infrastructure/test262/negative.js new file mode 100644 index 00000000000000..f7c8bb26242cb4 --- /dev/null +++ b/infrastructure/test262/negative.js @@ -0,0 +1,7 @@ +/*--- +description: A negative smoketest +negative: + phase: runtime + type: Test262Error +---*/ +throw new Test262Error(); diff --git a/infrastructure/test262/strict-error.js b/infrastructure/test262/strict-error.js new file mode 100644 index 00000000000000..7280d62c5d500b --- /dev/null +++ b/infrastructure/test262/strict-error.js @@ -0,0 +1,6 @@ +/*--- +description: A strict mode smoketest that throws an unexpected ReferenceError. +flags: [onlyStrict] +---*/ +"use strict"; +foo.bar(); diff --git a/infrastructure/test262/strict-failure.js b/infrastructure/test262/strict-failure.js new file mode 100644 index 00000000000000..57d302fbedad06 --- /dev/null +++ b/infrastructure/test262/strict-failure.js @@ -0,0 +1,6 @@ +/*--- +description: A strict mode smoketest that fails an assertion. +flags: [onlyStrict] +---*/ +"use strict"; +assert.sameValue(1, 2, "One should be two in strict mode"); diff --git a/infrastructure/test262/strict.js b/infrastructure/test262/strict.js new file mode 100644 index 00000000000000..aec5537b1c163c --- /dev/null +++ b/infrastructure/test262/strict.js @@ -0,0 +1,8 @@ +/*--- +description: A strict-mode smoketest from https://github.com/tc39/test262/blob/main/INTERPRETING.md +negative: + phase: parse + type: SyntaxError +---*/ +$DONOTEVALUATE(); +var a\u2E2F; diff --git a/infrastructure/test262/support/module-helper.js b/infrastructure/test262/support/module-helper.js new file mode 100644 index 00000000000000..cc798ff50da947 --- /dev/null +++ b/infrastructure/test262/support/module-helper.js @@ -0,0 +1 @@ +export const a = 1; diff --git a/infrastructure/test262/unexpected-error.js b/infrastructure/test262/unexpected-error.js new file mode 100644 index 00000000000000..4999a8b9450190 --- /dev/null +++ b/infrastructure/test262/unexpected-error.js @@ -0,0 +1,5 @@ +/*--- +description: A smoketest that throws an unexpected ReferenceError. +---*/ + +foo.bar(); diff --git a/lint.ignore b/lint.ignore index 1406c16ba218ae..586a11675b617a 100644 --- a/lint.ignore +++ b/lint.ignore @@ -841,3 +841,5 @@ PROMISE_REJECTS: wasm/core/js/harness/testharness.js # Legitimate use of test_driver_internal TEST DRIVER INTERNAL: resources/testdriver.js + +*: third_party/test262/* diff --git a/resources/test262/harness-adapter.js b/resources/test262/harness-adapter.js new file mode 100644 index 00000000000000..2de808bd6afe8f --- /dev/null +++ b/resources/test262/harness-adapter.js @@ -0,0 +1,173 @@ +function installAPI(global) { + global.$262 = { + createRealm: function() { + var iframe = global.document.createElement('iframe'); + iframe.style.cssText = 'display: none'; + iframe.src = ''; // iframeSrc; + if (global.document.body === null) { + global.document.body = global.document.createElement('body'); + } + global.document.body.appendChild(iframe); + return installAPI(iframe.contentWindow); + }, + evalScript: function(src) { + var script = global.document.createElement('script'); + script.text = src; + window.__test262_evalScript_error_ = undefined; + global.document.head.appendChild(script); + // Errors in the above appendChild bubble up to the global error handler. + // Our testharnes-client.js stashes them in a global var for rethrowing. + if (window.__test262_evalScript_error_) { + err = window.__test262_evalScript_error_; + window.__test262_evalScript_error_ = undefined; + throw err; + } + }, + detachArrayBuffer: function(buffer) { + if (typeof postMessage !== 'function') { + throw new Error('No method available to detach an ArrayBuffer'); + } else { + postMessage(null, '*', [buffer]); + /* + See + https://html.spec.whatwg.org/multipage/comms.html#dom-window-postmessage + which calls + https://html.spec.whatwg.org/multipage/infrastructure.html#structuredclonewithtransfer + which calls + https://html.spec.whatwg.org/multipage/infrastructure.html#transfer-abstract-op + which calls the DetachArrayBuffer abstract operation + https://tc39.github.io/ecma262/#sec-detacharraybuffer + */ + } + }, + gc: function() { + if (typeof gc !== 'function') { + throw new Error('No method available to invoke a GC'); + } + gc(); + }, + AbstractModuleSource: function() { + throw new Error('AbstractModuleSource not available'); + }, + agent: (function () { + var workers = []; + var i32a = null; + var pendingReports = []; + + // Agents call Atomics.wait on this location to sleep. + var SLEEP_LOC = 0; + // 1 if the started worker is ready, 0 otherwise. + var START_LOC = 1; + // The number of workers that have received the broadcast. + var BROADCAST_LOC = 2; + // Each worker has a count of outstanding reports; worker N uses memory + // location [WORKER_REPORT_LOC + N]. + var WORKER_REPORT_LOC = 3; + + function workerScript(script) { + return ` + var index; + var i32a = null; + var broadcasts = []; + var pendingReceiver = null; + + function handleBroadcast() { + if (pendingReceiver && broadcasts.length > 0) { + pendingReceiver.apply(null, broadcasts.shift()); + pendingReceiver = null; + } + }; + + var onmessage = function({data:msg}) { + switch (msg.kind) { + case 'start': + i32a = msg.i32a; + index = msg.index; + (0, eval)(\`${script}\`); + break; + + case 'broadcast': + Atomics.add(i32a, ${BROADCAST_LOC}, 1); + broadcasts.push([msg.sab, msg.id]); + handleBroadcast(); + break; + } + }; + + var $262 = { + agent: { + receiveBroadcast(receiver) { + pendingReceiver = receiver; + handleBroadcast(); + }, + + report(msg) { + postMessage(String(msg)); + Atomics.add(i32a, ${WORKER_REPORT_LOC} + index, 1); + }, + + sleep(s) { Atomics.wait(i32a, ${SLEEP_LOC}, 0, s); }, + + leaving() {}, + + monotonicNow() { + return performance.now(); + } + } + };`; + } + + var agent = { + start(script) { + if (i32a === null) { + i32a = new Int32Array(new SharedArrayBuffer(256)); + } + var w = new Worker(workerScript(script), {type: 'string'}); + w.index = workers.length; + w.postMessage({kind: 'start', i32a: i32a, index: w.index}); + workers.push(w); + }, + + broadcast(sab, id) { + if (!(sab instanceof SharedArrayBuffer)) { + throw new TypeError('sab must be a SharedArrayBuffer.'); + } + + Atomics.store(i32a, BROADCAST_LOC, 0); + + for (var w of workers) { + w.postMessage({kind: 'broadcast', sab: sab, id: id|0}); + } + + while (Atomics.load(i32a, BROADCAST_LOC) != workers.length) {} + }, + + getReport() { + for (var w of workers) { + while (Atomics.load(i32a, WORKER_REPORT_LOC + w.index) > 0) { + pendingReports.push(w.getMessage()); + Atomics.sub(i32a, WORKER_REPORT_LOC + w.index, 1); + } + } + + return pendingReports.shift() || null; + }, + + sleep(s) { Atomics.wait(i32a, SLEEP_LOC, 0, s); }, + + monotonicNow() { + return performance.now(); + } + }; + return agent; + + })(), + global: global + }; + global.$DONE = function() {} + + return global.$262; +} + +installAPI(window); + diff --git a/resources/test262/testharness-client.js b/resources/test262/testharness-client.js new file mode 100644 index 00000000000000..2014bcfa4a3e7f --- /dev/null +++ b/resources/test262/testharness-client.js @@ -0,0 +1,77 @@ +/* + * Minimalistic testharness-client tailored to run Test262 + * + * Expects parent document to listen for messages using + * test262/testharness.js. + * + */ + + +(function() { + // A placeholder until the real one is loaded from assert.js + function Test262Error(message) { + this.message = message; + } + Test262Error.prototype.name = "Test262Error"; + self.Test262Error = Test262Error; + + // We stash these in case the test overrides them + var Object_prototype_toString = Object.prototype.toString; + var Error_prototype_toString = Error.prototype.toString; + var String_prototype_indexOf = String.prototype.indexOf; + var parentWindow = window.parent; + + var expectedError; + var test_finished = false; + var status = 0; + var message = "OK"; + + window.test262Setup = function() { + } + + function done() { + if (test_finished) { return; } + test_finished = true; + parentWindow.postMessage(message, '*'); + parentWindow.postMessage(status, '*'); + } + window.test262Done = done; + + function on_error(event) { + // This hack ensures that errors thrown inside of a $262.evalScript get + // rethrown in the correct place. + if (event.error && String_prototype_indexOf.call(event.error.message, "Failed to execute 'appendChild' on 'Node'") === 0) { + window.__test262_evalScript_error_ = event.error; + return; + } + + if (expectedError && event.error && + (String_prototype_indexOf.call(event.error.toString(), expectedError) === 0 || + String_prototype_indexOf.call(Error_prototype_toString.call(event.error), expectedError) === 0 || + String_prototype_indexOf.call(event.message, expectedError) === 0)) { + status = 0; // OK + message = "OK"; + } else if (event.error instanceof self.Test262Error) { + status = 1; // FAIL + message = event.error.message; + } else { + status = 2; // ERROR + message = event.message; + } + done(); + } + window.addEventListener("error", on_error); + window.addEventListener("unhandledrejection", function(event) { + on_error({ + message: "Unhandled promise rejection: " + event.reason, + error: event.reason + }); + }); + + window.test262Negative = function(err) { + expectedError = err; + status = 1; + message = "Expected "+err; + window.$DONTEVALUATE = function() {}; + }; +})(); diff --git a/resources/test262/testharness.js b/resources/test262/testharness.js new file mode 100644 index 00000000000000..2554c25bb7d36c --- /dev/null +++ b/resources/test262/testharness.js @@ -0,0 +1,114 @@ +/* + * Minimalistic testharness tailored to run Test262 + * Can be used with or without testharnessreport.js. + * + * Expects actual test to be run in iframe and using + * test262/testharness-client.js. + * + */ +(function() { + var test_finished = false; + var callback; + + var harness_status = { + status: 0, + message: "OK" + }; + + function log(msg) { + document.getElementById('log').innerHTML += ""+msg+"
"; + } + + function report_result() { + log('done'); + log(JSON.stringify(harness_status)); + + if (callback || window.opener) { + var tests = [{ + name: document.title, + status: harness_status.subtest_status + }]; + var stat = { + status: harness_status.status, + message: harness_status.message, + stack: harness_status.stack + }; + var message = { + type: "complete", + tests: tests, + status: stat + }; + if (callback) { + log("callback"); + callback(tests, stat); + } else if (window.opener) { + log("postMessage"); + window.opener.postMessage(message, "*"); + } + return; + } + + // Fall-back support for running without testharnessreport.js included. + var retry = 0; + var result_payload = ["url", "complete", [ + 0, document.title, "", [] + ]]; + function raw_report() { + result_payload[0] = window.__wptrunner_url; + result_payload[2][0] = harness_status.status; + result_payload[2][2] = harness_status.message; + window.__wptrunner_testdriver_callback(result_payload); + window.__wptrunner_process_next_event(); + } + function call_raw_report() { + if (window.__wptrunner_testdriver_callback) { + return raw_report(); + } + retry += 1; + if (retry < 10) { + setTimeout(call_raw_report, 10); + } else { + log("could not communicate result"); + } + } + setTimeout(call_raw_report, 0); + log("raw fallback"); + } + + function done() { + if (test_finished) { return; } + test_finished = true; + report_result(); + } + + window.add_completion_callback = function(cb) { + callback = cb; + }; + window.setup = function() { + }; + + window.addEventListener('message', (event) => { + const iframe = document.getElementById('test262-iframe'); + // Communication protocol is to first receive a string message and then a + // result number. On purpose no objects since test262 mucks with the + // environment a lot (e.g., polluting Object.prototype and such). + if (iframe.contentWindow === event.source) { + if (typeof event.data === 'string') { + harness_status.message = event.data; + } + if (typeof event.data === 'number') { + if (event.data === 1) { // Test262Error, so a subtest failure + harness_status.status = 0; // Overall harness is OK + harness_status.subtest_status = 1; // Subtest is FAIL + } else if (event.data === 2) { // Other error, so a harness error + harness_status.status = 1; // Overall harness is ERROR + harness_status.subtest_status = 1; // Subtest is FAIL + } else { + harness_status.status = event.data; + harness_status.subtest_status = event.data; + } + done(); + } + } + }); +})(); diff --git a/third_party/test262/harness/assert.js b/third_party/test262/harness/assert.js new file mode 100644 index 00000000000000..7ced0cd0b4129f --- /dev/null +++ b/third_party/test262/harness/assert.js @@ -0,0 +1,170 @@ +// Copyright (C) 2017 Ecma International. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +description: | + Collection of assertion functions used throughout test262 +defines: [assert] +---*/ + + +function assert(mustBeTrue, message) { + if (mustBeTrue === true) { + return; + } + + if (message === undefined) { + message = 'Expected true but got ' + assert._toString(mustBeTrue); + } + throw new Test262Error(message); +} + +assert._isSameValue = function (a, b) { + if (a === b) { + // Handle +/-0 vs. -/+0 + return a !== 0 || 1 / a === 1 / b; + } + + // Handle NaN vs. NaN + return a !== a && b !== b; +}; + +assert.sameValue = function (actual, expected, message) { + try { + if (assert._isSameValue(actual, expected)) { + return; + } + } catch (error) { + throw new Test262Error(message + ' (_isSameValue operation threw) ' + error); + return; + } + + if (message === undefined) { + message = ''; + } else { + message += ' '; + } + + message += 'Expected SameValue(«' + assert._toString(actual) + '», «' + assert._toString(expected) + '») to be true'; + + throw new Test262Error(message); +}; + +assert.notSameValue = function (actual, unexpected, message) { + if (!assert._isSameValue(actual, unexpected)) { + return; + } + + if (message === undefined) { + message = ''; + } else { + message += ' '; + } + + message += 'Expected SameValue(«' + assert._toString(actual) + '», «' + assert._toString(unexpected) + '») to be false'; + + throw new Test262Error(message); +}; + +assert.throws = function (expectedErrorConstructor, func, message) { + var expectedName, actualName; + if (typeof func !== "function") { + throw new Test262Error('assert.throws requires two arguments: the error constructor ' + + 'and a function to run'); + return; + } + if (message === undefined) { + message = ''; + } else { + message += ' '; + } + + try { + func(); + } catch (thrown) { + if (typeof thrown !== 'object' || thrown === null) { + message += 'Thrown value was not an object!'; + throw new Test262Error(message); + } else if (thrown.constructor !== expectedErrorConstructor) { + expectedName = expectedErrorConstructor.name; + actualName = thrown.constructor.name; + if (expectedName === actualName) { + message += 'Expected a ' + expectedName + ' but got a different error constructor with the same name'; + } else { + message += 'Expected a ' + expectedName + ' but got a ' + actualName; + } + throw new Test262Error(message); + } + return; + } + + message += 'Expected a ' + expectedErrorConstructor.name + ' to be thrown but no exception was thrown at all'; + throw new Test262Error(message); +}; + +function isPrimitive(value) { + return !value || (typeof value !== 'object' && typeof value !== 'function'); +} + +assert.compareArray = function (actual, expected, message) { + message = message === undefined ? '' : message; + + if (typeof message === 'symbol') { + message = message.toString(); + } + + if (isPrimitive(actual)) { + assert(false, `Actual argument [${actual}] shouldn't be primitive. ${message}`); + } else if (isPrimitive(expected)) { + assert(false, `Expected argument [${expected}] shouldn't be primitive. ${message}`); + } + var result = compareArray(actual, expected); + if (result) return; + + var format = compareArray.format; + assert(false, `Actual ${format(actual)} and expected ${format(expected)} should have the same contents. ${message}`); +}; + +function compareArray(a, b) { + if (b.length !== a.length) { + return false; + } + for (var i = 0; i < a.length; i++) { + if (!assert._isSameValue(b[i], a[i])) { + return false; + } + } + return true; +} + +compareArray.format = function (arrayLike) { + return `[${Array.prototype.map.call(arrayLike, String).join(', ')}]`; +}; + +assert._formatIdentityFreeValue = function formatIdentityFreeValue(value) { + switch (value === null ? 'null' : typeof value) { + case 'string': + return typeof JSON !== "undefined" ? JSON.stringify(value) : `"${value}"`; + case 'bigint': + return `${value}n`; + case 'number': + if (value === 0 && 1 / value === -Infinity) return '-0'; + // falls through + case 'boolean': + case 'undefined': + case 'null': + return String(value); + } +}; + +assert._toString = function (value) { + var basic = assert._formatIdentityFreeValue(value); + if (basic) return basic; + try { + return String(value); + } catch (err) { + if (err.name === 'TypeError') { + return Object.prototype.toString.call(value); + } + throw err; + } +}; diff --git a/third_party/test262/harness/sta.js b/third_party/test262/harness/sta.js new file mode 100644 index 00000000000000..c95ed7a264c550 --- /dev/null +++ b/third_party/test262/harness/sta.js @@ -0,0 +1,27 @@ +// Copyright (c) 2012 Ecma International. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +description: | + Provides both: + + - An error class to avoid false positives when testing for thrown exceptions + - A function to explicitly throw an exception using the Test262Error class +defines: [Test262Error, $DONOTEVALUATE] +---*/ + + +function Test262Error(message) { + this.message = message || ""; +} + +Test262Error.prototype.toString = function () { + return "Test262Error: " + this.message; +}; + +Test262Error.thrower = function (message) { + throw new Test262Error(message); +}; + +function $DONOTEVALUATE() { + throw "Test262: This statement should not be evaluated."; +} diff --git a/tools/ci/tc/tasks/test.yml b/tools/ci/tc/tasks/test.yml index daa86a629f2924..0c74482e51f6b3 100644 --- a/tools/ci/tc/tasks/test.yml +++ b/tools/ci/tc/tasks/test.yml @@ -41,6 +41,12 @@ components: vars: test-type: crashtest + wpt-test262: + chunks: 16 + vars: + test-type: test262 + + run-options: options: xvfb: true @@ -157,6 +163,8 @@ tasks: suite: wdspec - vars: suite: crashtest + - vars: + suite: test262 do: $map: for: diff --git a/tools/ci/tc/tests/test_valid.py b/tools/ci/tc/tests/test_valid.py index 4a12ff8927c34d..d54bfcf973601f 100644 --- a/tools/ci/tc/tests/test_valid.py +++ b/tools/ci/tc/tests/test_valid.py @@ -159,6 +159,38 @@ def test_verify_payload(): 'wpt-chrome-canary-wdspec-2', 'wpt-firefox-nightly-crashtest-1', 'wpt-chrome-canary-crashtest-1', + 'wpt-firefox-nightly-test262-1', + 'wpt-firefox-nightly-test262-2', + 'wpt-firefox-nightly-test262-3', + 'wpt-firefox-nightly-test262-4', + 'wpt-firefox-nightly-test262-5', + 'wpt-firefox-nightly-test262-6', + 'wpt-firefox-nightly-test262-7', + 'wpt-firefox-nightly-test262-8', + 'wpt-firefox-nightly-test262-9', + 'wpt-firefox-nightly-test262-10', + 'wpt-firefox-nightly-test262-11', + 'wpt-firefox-nightly-test262-12', + 'wpt-firefox-nightly-test262-13', + 'wpt-firefox-nightly-test262-14', + 'wpt-firefox-nightly-test262-15', + 'wpt-firefox-nightly-test262-16', + 'wpt-chrome-canary-test262-1', + 'wpt-chrome-canary-test262-2', + 'wpt-chrome-canary-test262-3', + 'wpt-chrome-canary-test262-4', + 'wpt-chrome-canary-test262-5', + 'wpt-chrome-canary-test262-6', + 'wpt-chrome-canary-test262-7', + 'wpt-chrome-canary-test262-8', + 'wpt-chrome-canary-test262-9', + 'wpt-chrome-canary-test262-10', + 'wpt-chrome-canary-test262-11', + 'wpt-chrome-canary-test262-12', + 'wpt-chrome-canary-test262-13', + 'wpt-chrome-canary-test262-14', + 'wpt-chrome-canary-test262-15', + 'wpt-chrome-canary-test262-16', 'wpt-firefox-nightly-reftest-1', 'wpt-firefox-nightly-reftest-2', 'wpt-firefox-nightly-reftest-3', @@ -389,6 +421,134 @@ def test_verify_payload(): 'wpt-servo-nightly-crashtest-1', 'wpt-firefox_android-nightly-crashtest-1', 'wpt-firefox_android-stable-crashtest-1', + 'wpt-firefox-stable-test262-1', + 'wpt-firefox-stable-test262-2', + 'wpt-firefox-stable-test262-3', + 'wpt-firefox-stable-test262-4', + 'wpt-firefox-stable-test262-5', + 'wpt-firefox-stable-test262-6', + 'wpt-firefox-stable-test262-7', + 'wpt-firefox-stable-test262-8', + 'wpt-firefox-stable-test262-9', + 'wpt-firefox-stable-test262-10', + 'wpt-firefox-stable-test262-11', + 'wpt-firefox-stable-test262-12', + 'wpt-firefox-stable-test262-13', + 'wpt-firefox-stable-test262-14', + 'wpt-firefox-stable-test262-15', + 'wpt-firefox-stable-test262-16', + 'wpt-chromium-nightly-test262-1', + 'wpt-chromium-nightly-test262-2', + 'wpt-chromium-nightly-test262-3', + 'wpt-chromium-nightly-test262-4', + 'wpt-chromium-nightly-test262-5', + 'wpt-chromium-nightly-test262-6', + 'wpt-chromium-nightly-test262-7', + 'wpt-chromium-nightly-test262-8', + 'wpt-chromium-nightly-test262-9', + 'wpt-chromium-nightly-test262-10', + 'wpt-chromium-nightly-test262-11', + 'wpt-chromium-nightly-test262-12', + 'wpt-chromium-nightly-test262-13', + 'wpt-chromium-nightly-test262-14', + 'wpt-chromium-nightly-test262-15', + 'wpt-chromium-nightly-test262-16', + 'wpt-chrome-stable-test262-1', + 'wpt-chrome-stable-test262-2', + 'wpt-chrome-stable-test262-3', + 'wpt-chrome-stable-test262-4', + 'wpt-chrome-stable-test262-5', + 'wpt-chrome-stable-test262-6', + 'wpt-chrome-stable-test262-7', + 'wpt-chrome-stable-test262-8', + 'wpt-chrome-stable-test262-9', + 'wpt-chrome-stable-test262-10', + 'wpt-chrome-stable-test262-11', + 'wpt-chrome-stable-test262-12', + 'wpt-chrome-stable-test262-13', + 'wpt-chrome-stable-test262-14', + 'wpt-chrome-stable-test262-15', + 'wpt-chrome-stable-test262-16', + 'wpt-webkitgtk_minibrowser-nightly-test262-1', + 'wpt-webkitgtk_minibrowser-nightly-test262-2', + 'wpt-webkitgtk_minibrowser-nightly-test262-3', + 'wpt-webkitgtk_minibrowser-nightly-test262-4', + 'wpt-webkitgtk_minibrowser-nightly-test262-5', + 'wpt-webkitgtk_minibrowser-nightly-test262-6', + 'wpt-webkitgtk_minibrowser-nightly-test262-7', + 'wpt-webkitgtk_minibrowser-nightly-test262-8', + 'wpt-webkitgtk_minibrowser-nightly-test262-9', + 'wpt-webkitgtk_minibrowser-nightly-test262-10', + 'wpt-webkitgtk_minibrowser-nightly-test262-11', + 'wpt-webkitgtk_minibrowser-nightly-test262-12', + 'wpt-webkitgtk_minibrowser-nightly-test262-13', + 'wpt-webkitgtk_minibrowser-nightly-test262-14', + 'wpt-webkitgtk_minibrowser-nightly-test262-15', + 'wpt-webkitgtk_minibrowser-nightly-test262-16', + 'wpt-wpewebkit_minibrowser-nightly-test262-1', + 'wpt-wpewebkit_minibrowser-nightly-test262-2', + 'wpt-wpewebkit_minibrowser-nightly-test262-3', + 'wpt-wpewebkit_minibrowser-nightly-test262-4', + 'wpt-wpewebkit_minibrowser-nightly-test262-5', + 'wpt-wpewebkit_minibrowser-nightly-test262-6', + 'wpt-wpewebkit_minibrowser-nightly-test262-7', + 'wpt-wpewebkit_minibrowser-nightly-test262-8', + 'wpt-wpewebkit_minibrowser-nightly-test262-9', + 'wpt-wpewebkit_minibrowser-nightly-test262-10', + 'wpt-wpewebkit_minibrowser-nightly-test262-11', + 'wpt-wpewebkit_minibrowser-nightly-test262-12', + 'wpt-wpewebkit_minibrowser-nightly-test262-13', + 'wpt-wpewebkit_minibrowser-nightly-test262-14', + 'wpt-wpewebkit_minibrowser-nightly-test262-15', + 'wpt-wpewebkit_minibrowser-nightly-test262-16', + 'wpt-servo-nightly-test262-1', + 'wpt-servo-nightly-test262-2', + 'wpt-servo-nightly-test262-3', + 'wpt-servo-nightly-test262-4', + 'wpt-servo-nightly-test262-5', + 'wpt-servo-nightly-test262-6', + 'wpt-servo-nightly-test262-7', + 'wpt-servo-nightly-test262-8', + 'wpt-servo-nightly-test262-9', + 'wpt-servo-nightly-test262-10', + 'wpt-servo-nightly-test262-11', + 'wpt-servo-nightly-test262-12', + 'wpt-servo-nightly-test262-13', + 'wpt-servo-nightly-test262-14', + 'wpt-servo-nightly-test262-15', + 'wpt-servo-nightly-test262-16', + 'wpt-firefox_android-nightly-test262-1', + 'wpt-firefox_android-nightly-test262-2', + 'wpt-firefox_android-nightly-test262-3', + 'wpt-firefox_android-nightly-test262-4', + 'wpt-firefox_android-nightly-test262-5', + 'wpt-firefox_android-nightly-test262-6', + 'wpt-firefox_android-nightly-test262-7', + 'wpt-firefox_android-nightly-test262-8', + 'wpt-firefox_android-nightly-test262-9', + 'wpt-firefox_android-nightly-test262-10', + 'wpt-firefox_android-nightly-test262-11', + 'wpt-firefox_android-nightly-test262-12', + 'wpt-firefox_android-nightly-test262-13', + 'wpt-firefox_android-nightly-test262-14', + 'wpt-firefox_android-nightly-test262-15', + 'wpt-firefox_android-nightly-test262-16', + 'wpt-firefox_android-stable-test262-1', + 'wpt-firefox_android-stable-test262-2', + 'wpt-firefox_android-stable-test262-3', + 'wpt-firefox_android-stable-test262-4', + 'wpt-firefox_android-stable-test262-5', + 'wpt-firefox_android-stable-test262-6', + 'wpt-firefox_android-stable-test262-7', + 'wpt-firefox_android-stable-test262-8', + 'wpt-firefox_android-stable-test262-9', + 'wpt-firefox_android-stable-test262-10', + 'wpt-firefox_android-stable-test262-11', + 'wpt-firefox_android-stable-test262-12', + 'wpt-firefox_android-stable-test262-13', + 'wpt-firefox_android-stable-test262-14', + 'wpt-firefox_android-stable-test262-15', + 'wpt-firefox_android-stable-test262-16', 'wpt-firefox-stable-reftest-1', 'wpt-firefox-stable-reftest-2', 'wpt-firefox-stable-reftest-3', diff --git a/tools/manifest/__init__.py b/tools/manifest/__init__.py index 8c8f189070eaa3..bc0624ae417283 100644 --- a/tools/manifest/__init__.py +++ b/tools/manifest/__init__.py @@ -1 +1 @@ -from . import item, manifest, sourcefile, update # noqa: F401 +from . import item, manifest, sourcefile, test262, update # noqa: F401 diff --git a/tools/manifest/commands.json b/tools/manifest/commands.json index cef6d22473b0fb..80b9432f095a49 100644 --- a/tools/manifest/commands.json +++ b/tools/manifest/commands.json @@ -4,7 +4,10 @@ "script": "run", "parser": "create_parser", "help": "Update the MANIFEST.json file", - "virtualenv": false + "virtualenv": true, + "requirements": [ + "requirements.txt" + ] }, "manifest-download": { "path": "download.py", diff --git a/tools/manifest/item.py b/tools/manifest/item.py index 99df09d1320dc9..6aa9f6c8e68347 100644 --- a/tools/manifest/item.py +++ b/tools/manifest/item.py @@ -198,6 +198,12 @@ def to_json(self) -> Tuple[Optional[Text], Dict[Text, Any]]: return rv +class Test262Test(TestharnessTest): + __slots__ = () + + item_type = "test262" + + class RefTest(URLManifestItem): __slots__ = ("references",) diff --git a/tools/manifest/manifest.py b/tools/manifest/manifest.py index c4eca5f26eb77c..12260192c3297c 100644 --- a/tools/manifest/manifest.py +++ b/tools/manifest/manifest.py @@ -17,6 +17,7 @@ SpecItem, SupportFile, TestharnessTest, + Test262Test, VisualTest, WebDriverSpecTest) from .log import get_logger @@ -49,7 +50,8 @@ class InvalidCacheError(Exception): "conformancechecker": ConformanceCheckerTest, "visual": VisualTest, "spec": SpecItem, - "support": SupportFile} + "support": SupportFile, + "test262": Test262Test} def compute_manifest_items(source_file: SourceFile) -> Optional[Tuple[Tuple[Text, ...], Text, Set[ManifestItem], Text]]: diff --git a/tools/manifest/requirements.txt b/tools/manifest/requirements.txt index ca872b12c41955..d12c93a75afe14 100644 --- a/tools/manifest/requirements.txt +++ b/tools/manifest/requirements.txt @@ -1 +1,3 @@ zstandard==0.23.0 +pyyaml==6.0.1 +types-pyyaml==6.0.12.20241230 diff --git a/tools/manifest/sourcefile.py b/tools/manifest/sourcefile.py index 3628105006c972..45d960d540aced 100644 --- a/tools/manifest/sourcefile.py +++ b/tools/manifest/sourcefile.py @@ -24,11 +24,14 @@ RefTest, SpecItem, SupportFile, + Test262Test, TestharnessTest, VisualTest, WebDriverSpecTest) from .utils import cached_property +from . import test262 + # Cannot do `from ..metadata.webfeatures.schema import WEB_FEATURES_YML_FILENAME` # because relative import beyond toplevel throws *ImportError*! from metadata.webfeatures.schema import WEB_FEATURES_YML_FILENAME # type: ignore @@ -421,6 +424,12 @@ def name_is_print_reftest(self) -> bool: return (self.markup_type is not None and (self.type_flag == "print" or "print" in self.dir_path.split(os.path.sep))) + @property + def name_is_test262(self) -> bool: + """Check if the file name matches the conditions for the file to be a + test262 file""" + return ("test262" in self.dir_path.split(os.path.sep) and self.ext == ".js") + @property def markup_type(self) -> Optional[Text]: """Return the type of markup contained in a file, based on its extension, @@ -470,12 +479,30 @@ def pac_nodes(self) -> List[ElementTree.Element]: assert self.root is not None return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='pac']") + @cached_property + def test262_test_record(self) -> Optional[Dict[Text, Any]]: + if self.name_is_test262: + with self.open() as f: + return test262.TestRecord.parse(f.read().decode('ISO-8859-1'), self.path) + else: + return None + @cached_property def script_metadata(self) -> Optional[List[Tuple[Text, Text]]]: if self.name_is_worker or self.name_is_multi_global or self.name_is_window or self.name_is_extension: regexp = js_meta_re elif self.name_is_webdriver: regexp = python_meta_re + elif self.name_is_test262: + if self.test262_test_record is None: + return None + paths = [] + for filename in self.test262_test_record.get("includes", []): + if filename in ("assert.js", "sta.js"): + paths.append(('script', "/third_party/test262/harness/%s" % filename)) + else: + paths.append(('script', "/resources/test262/%s" % filename)) + return paths else: return None @@ -917,6 +944,9 @@ def possible_types(self) -> Set[Text]: if self.name_is_window: return {TestharnessTest.item_type} + if self.name_is_test262: + return {Test262Test.item_type, SupportFile.item_type} + if self.name_is_extension: return {TestharnessTest.item_type} @@ -1100,6 +1130,38 @@ def manifest_items(self) -> Tuple[Text, List[ManifestItem]]: ] rv = TestharnessTest.item_type, tests + elif self.name_is_test262: + if self.test262_test_record is None: + rv = "support", [ + SupportFile( + self.tests_root, + self.rel_path + )] + else: + suffix = ".test262" + if "module" in self.test262_test_record: + suffix += "-module" + elif "onlyStrict" in self.test262_test_record: + # Modules are always strict mode, so only append strict for + # non-module tests. + suffix += ".strict" + suffix += ".html" + test_url = replace_end(self.rel_url, ".js", suffix) + tests = [ + Test262Test( + self.tests_root, + self.rel_path, + self.url_base, + test_url + variant, + timeout=self.timeout, + pac=self.pac, + testdriver_features=self.testdriver_features, + script_metadata=self.script_metadata + ) + for variant in self.test_variants + ] + rv = Test262Test.item_type, tests + elif self.content_is_css_manual and not self.name_is_reference: rv = ManualTest.item_type, [ ManualTest( diff --git a/tools/manifest/test262.py b/tools/manifest/test262.py new file mode 100644 index 00000000000000..cfb110d60d4ead --- /dev/null +++ b/tools/manifest/test262.py @@ -0,0 +1,62 @@ +from __future__ import print_function + +from typing import Dict, Optional, Text, Tuple, Any, Callable + +import re + +# Matches trailing whitespace and any following blank lines. +_BLANK_LINES = r"([ \t]*[\r\n]{1,2})*" + +# Matches the YAML frontmatter block. +_YAML_PATTERN = re.compile(r"/\*---(.*)---\*/" + _BLANK_LINES, re.DOTALL) + +_STRIP_CONTROL_CHARS = re.compile(r'[\x7f-\x9f]') + + +class TestRecord: + @staticmethod + def _yaml_attr_parser(test_record: Dict[Text, Any], attrs: Text, name: Text, onerror: Callable[[Text], Any]) -> None: + import yaml + parsed = yaml.safe_load(re.sub(_STRIP_CONTROL_CHARS, ' ', attrs)) + if parsed is None: + onerror("Failed to parse yaml in name %s" % name) + return + + for key in parsed: + value = parsed[key] + if key == "info": + key = "commentary" + test_record[key] = value + + if 'flags' in test_record: + for flag in test_record['flags']: + test_record[flag] = "" + + @staticmethod + def _find_attrs(src: Text) -> Tuple[Optional[Text], Optional[Text]]: + match = _YAML_PATTERN.search(src) + if not match: + return (None, None) + + return (match.group(0), match.group(1).strip()) + + @staticmethod + def parse(src: Text, name: Text, onerror: Callable[[Text], Any] = print) -> Optional[Dict[Text, Any]]: + if name.endswith('_FIXTURE.js'): + return None + + # Find the YAML frontmatter. + (frontmatter, attrs) = TestRecord._find_attrs(src) + + # YAML frontmatter is required for all tests. + if frontmatter is None: + onerror("Missing frontmatter: %s" % name) + return None + + test_record: Dict[Text, Any] = {} + test_record['test'] = src + + if attrs: + TestRecord._yaml_attr_parser(test_record, attrs, name, onerror) + + return test_record diff --git a/tools/manifest/tests/test_sourcefile.py b/tools/manifest/tests/test_sourcefile.py index 4859030c2de8ed..816aa37d8a9296 100644 --- a/tools/manifest/tests/test_sourcefile.py +++ b/tools/manifest/tests/test_sourcefile.py @@ -1013,3 +1013,40 @@ def test_html_testdriver_features(features): s = create("html/test.html", contents=contents) assert s.testdriver_features == features + +@pytest.mark.parametrize("rel_path, is_test262", [ + ("test262/test.js", True), + ("other/test.js", False), +]) +def test_name_is_test262(rel_path, is_test262): + tests_root = "/tmp" + url_base = "/" + sf = SourceFile(tests_root, rel_path, url_base) + assert sf.name_is_test262 == is_test262 + +def test_test262_test_record(): + contents = b"""/*--- +description: A simple test +---*/""" + sf = create("test262/test.js", contents=contents) + record = sf.test262_test_record + assert record is not None + assert record["description"] == "A simple test" + +@pytest.mark.parametrize("rel_path, contents, expected_url", [ + ("test262/test.js", + b"/*---\ndescription: A simple test\n---*/", + "/test262/test.test262.html"), + ("test262/module.js", + b"/*---\ndescription: A module test\nflags: [module]\n---*/", + "/test262/module.test262-module.html"), + ("test262/strict.js", + b"/*---\ndescription: A strict mode test\nflags: [onlyStrict]\n---*/", + "/test262/strict.test262.strict.html"), +]) +def test_manifest_items_test262(rel_path, contents, expected_url): + sf = create(rel_path, contents=contents) + item_type, items = sf.manifest_items() + assert item_type == "test262" + assert len(items) == 1 + assert items[0].url == expected_url diff --git a/tools/manifest/tests/test_test262.py b/tools/manifest/tests/test_test262.py new file mode 100644 index 00000000000000..65f717a605f960 --- /dev/null +++ b/tools/manifest/tests/test_test262.py @@ -0,0 +1,75 @@ +# mypy: allow-untyped-defs + +import pytest + +from tools.manifest.test262 import TestRecord + +@pytest.mark.parametrize("name, src, expected_record", [ + ( + "test.js", + """/*--- +description: A simple test +features: [Test262] +---*/ +assert.sameValue(1, 1); +""", + {"description": "A simple test", "features": ["Test262"]} + ), + ( + "no_frontmatter.js", + """assert.sameValue(1, 1);""", + None + ), + ( + "test_FIXTURE.js", + """/*--- +description: A fixture file +---*/ +assert.sameValue(1, 1); +""", + None + ), + ( + "flags.js", + """/*--- +description: Test with flags +flags: [raw, module] +---*/ +assert.sameValue(1, 1); +""", + {"flags": ["raw", "module"]} + ), + ( + "negative.js", + """/*--- +description: Negative test +negative: + phase: runtime + type: TypeError +---*/ +throw new TypeError(); +""", + {"negative": {"phase": "runtime", "type": "TypeError"}} + ), + ( + "includes.js", + """/*--- +description: Test with includes +includes: [assert.js, sta.js] +---*/ +assert.sameValue(1, 1); +""", + {"includes": ["assert.js", "sta.js"]} + ), +]) +def test_test262_parser(name, src, expected_record): + record = TestRecord.parse(src, name) + + if expected_record is None: + assert record is None + else: + assert record is not None + for key, value in expected_record.items(): + assert key in record + assert record[key] == value + assert record["test"] == src diff --git a/tools/serve/serve.py b/tools/serve/serve.py index e8c42129702ae1..d317730d1ae2db 100644 --- a/tools/serve/serve.py +++ b/tools/serve/serve.py @@ -19,11 +19,12 @@ from io import IOBase from itertools import chain, product from html5lib import html5parser -from typing import ClassVar, List, Optional, Set, Tuple +from typing import ClassVar, List, Optional, Set, Tuple, Union from localpaths import repo_root # type: ignore from manifest.sourcefile import read_script_metadata, js_meta_re, parse_variants # type: ignore +from manifest.test262 import TestRecord # type: ignore from wptserve import server as wptserve, handlers from wptserve import stash from wptserve import config @@ -327,6 +328,74 @@ class ExtensionHandler(HtmlWrapperHandler): """ +class Test262WindowHandler(HtmlWrapperHandler): + path_replace = [(".test262.html", ".js", ".test262-test.html")] + wrapper = """ + +Test + + +%(meta)s +%(script)s +
+""" + + +class Test262WindowTestHandler(HtmlWrapperHandler): + # For SHAB + headers = [('Cross-Origin-Opener-Policy', 'same-origin'), + ('Cross-Origin-Embedder-Policy', 'require-corp')] + + path_replace: Union[List[Tuple[str, str]], List[Tuple[str, str, str]]] = [(".test262-test.html", ".js")] + + pre_wrapper = """ + +Test + + + + +%(meta)s +%(script)s""" + wrapper = pre_wrapper + """ + +""" + + def _get_metadata(self, request): + path = self._get_filesystem_path(request) + with open(path, encoding='ISO-8859-1') as f: + test_record = TestRecord.parse(f.read(), path) + yield from (('script', "/third_party/test262/harness/%s" % filename) + for filename in test_record.get("includes", [])) + expected_error = test_record.get('negative', {}).get('type', None) + if expected_error is not None: + yield ('negative', expected_error) + + def _meta_replacement(self, key: str, value: str) -> Optional[str]: + if key == 'negative': + return """""" % value + return None + + +class Test262WindowModuleHandler(Test262WindowHandler): + path_replace = [(".test262-module.html", ".js", ".test262-module-test.html")] + +class Test262WindowModuleTestHandler(Test262WindowTestHandler): + path_replace = [(".test262-module-test.html", ".js")] + wrapper = Test262WindowTestHandler.pre_wrapper + """""" + + +class Test262StrictWindowHandler(Test262WindowHandler): + path_replace = [(".test262.strict.html", ".js", ".test262-test.strict.html")] + +class Test262StrictWindowTestHandler(Test262WindowTestHandler): + path_replace = [(".test262-test.strict.html", ".js", ".test262.strict.js")] + + class WindowModulesHandler(HtmlWrapperHandler): global_type = "window-module" path_replace = [(".any.window-module.html", ".any.js")] @@ -574,6 +643,31 @@ class ShadowRealmInAudioWorkletHandler(HtmlWrapperHandler): """ +class Test262StrictHandler(WrapperHandler): + path_replace = [(".test262.strict.js", ".js")] + headers = [('Content-Type', 'text/javascript')] + wrapper = """ +"use strict"; +%(script)s +""" + + def _meta_replacement(self, key, value): + return None + + def _get_metadata(self, request): + # Abuse the script metadata to inline the script content so as to + # prepend "use strict". + path = self._get_filesystem_path(request) + try: + with open(path, encoding='ISO-8859-1') as f: + yield ('script', f.read()) + except OSError: + raise HTTPException(404) + + def _script_replacement(self, key, value): + return value + + class BaseWorkerHandler(WrapperHandler): headers = [("Content-Type", "text/javascript")] @@ -787,6 +881,13 @@ def add_mount_point(self, url_base, path): ("GET", "*.worker.html", WorkersHandler), ("GET", "*.worker-module.html", WorkerModulesHandler), ("GET", "*.window.html", WindowHandler), + ("GET", "*.test262.html", Test262WindowHandler), + ("GET", "*.test262-test.html", Test262WindowTestHandler), + ("GET", "*.test262-module.html", Test262WindowModuleHandler), + ("GET", "*.test262-module-test.html", Test262WindowModuleTestHandler), + ("GET", "*.test262.strict.html", Test262StrictWindowHandler), + ("GET", "*.test262-test.strict.html", Test262StrictWindowTestHandler), + ("GET", "*.test262.strict.js", Test262StrictHandler), ("GET", "*.extension.html", ExtensionHandler), ("GET", "*.any.html", AnyHtmlHandler), ("GET", "*.any.sharedworker.html", SharedWorkersHandler), diff --git a/tools/serve/test_serve.py b/tools/serve/test_serve.py index 9dcda584de2ea4..1c4598994e50ce 100644 --- a/tools/serve/test_serve.py +++ b/tools/serve/test_serve.py @@ -1,15 +1,30 @@ # mypy: allow-untyped-defs +import builtins +import io import logging import os import pickle import platform +from unittest.mock import MagicMock, patch +from typing import Generator, List, Tuple, Type import pytest import localpaths # type: ignore from . import serve -from .serve import ConfigBuilder, inject_script +from .serve import ( + ConfigBuilder, + WrapperHandler, + inject_script, + # Use 'T262' aliases to avoid naming collisions with the pytest collector + Test262WindowHandler as T262WindowHandler, + Test262WindowTestHandler as T262WindowTestHandler, + Test262WindowModuleHandler as T262WindowModuleHandler, + Test262WindowModuleTestHandler as T262WindowModuleTestHandler, + Test262StrictWindowHandler as T262StrictWindowHandler, + Test262StrictWindowTestHandler as T262StrictWindowTestHandler, + Test262StrictHandler as T262StrictHandler) logger = logging.getLogger() @@ -154,3 +169,181 @@ def test_inject_script_parse_error(): # On a parse error, the script should not be injected and the original content should be # returned. assert INJECT_SCRIPT_MARKER not in inject_script(html.replace(INJECT_SCRIPT_MARKER, b""), INJECT_SCRIPT_MARKER) + + +@pytest.fixture +def test262_handlers() -> Generator[Tuple[str, str], None, None]: + tests_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "testdata")) + url_base = "/" + + mock_file_contents = { + os.path.normpath(os.path.join(tests_root, "test262", "basic.js")): """/*---\ndescription: A basic test +includes: [assert.js, sta.js] +---*/ +assert.sameValue(1, 1); +""", + os.path.normpath(os.path.join(tests_root, "test262", "negative.js")): """/*---\ndescription: A negative test +negative: + phase: runtime + type: TypeError +---*/ +throw new TypeError(); +""", + os.path.normpath(os.path.join(tests_root, "test262", "module.js")): """/*---\ndescription: A module test +flags: [module] +---*/ +import {} from 'some-module'; +""", + os.path.normpath(os.path.join(tests_root, "test262", "teststrict.js")): """/*---\ndescription: A strict mode test +flags: [onlyStrict] +includes: [propertyHelper.js] +---*/ +console.log('hello'); +""" + } + + # Store original functions to be called if our mock doesn't handle the file + original_open = builtins.open + original_exists = os.path.exists + original_isdir = os.path.isdir + + def custom_open(file, mode='r', *args, **kwargs): + normalized_file = os.path.normpath(file) + if normalized_file in mock_file_contents: + if 'b' in mode: + return io.BytesIO(mock_file_contents[normalized_file].encode('ISO-8859-1')) + else: + return io.StringIO(mock_file_contents[normalized_file]) + return original_open(file, mode, *args, **kwargs) + + def custom_exists(path): + normalized_path = os.path.normpath(path) + return normalized_path in mock_file_contents or original_exists(path) + + def custom_isdir(path): + normalized_path = os.path.normpath(path) + expected_dir = os.path.normpath(os.path.join(tests_root, "test262")) + return normalized_path == expected_dir or original_isdir(path) + + with patch('builtins.open', side_effect=custom_open), \ + patch('os.path.exists', side_effect=custom_exists), \ + patch('os.path.isdir', side_effect=custom_isdir): + yield tests_root, url_base + + +def _create_mock_request(path: str) -> MagicMock: + mock_request = MagicMock() + mock_request.url_parts.path = path + mock_request.url_parts.query = "" + return mock_request + + +def _test_handler_path_replace(handler_cls: Type[WrapperHandler], + tests_root: str, + url_base: str, + expected: List[Tuple[str, str]]) -> None: + handler = handler_cls(base_path=tests_root, url_base=url_base) + assert handler.path_replace == expected + +def _test_handler_wrapper_content(handler_cls: Type[WrapperHandler], + tests_root: str, + url_base: str, + request_path: str, + expected_content: List[str]) -> None: + handler = handler_cls(base_path=tests_root, url_base=url_base) + mock_request = _create_mock_request(request_path) + mock_response = MagicMock() + handler.handle_request(mock_request, mock_response) # type: ignore[no-untyped-call] + content = mock_response.content + for item in expected_content: + assert item in content + +def _test_handler_get_metadata(handler_cls: Type[WrapperHandler], + tests_root: str, + url_base: str, + request_path: str, + expected_metadata: List[Tuple[str, str]]) -> None: + handler = handler_cls(tests_root, url_base) + mock_request = _create_mock_request(request_path) + metadata = list(handler._get_metadata(mock_request)) # type: ignore[no-untyped-call] + for item in expected_metadata: + assert item in metadata + assert len(expected_metadata) == len(metadata), f"{expected_metadata} != {metadata}" + + +@pytest.mark.parametrize("handler_cls, expected", [ + (T262WindowHandler, [(".test262.html", ".js", ".test262-test.html")]), + (T262WindowTestHandler, [(".test262-test.html", ".js")]), + (T262WindowModuleHandler, [(".test262-module.html", ".js", ".test262-module-test.html")]), + (T262WindowModuleTestHandler, [(".test262-module-test.html", ".js")]), + (T262StrictWindowHandler, [(".test262.strict.html", ".js", ".test262-test.strict.html")]), + (T262StrictWindowTestHandler, [(".test262-test.strict.html", ".js", ".test262.strict.js")]), +]) +def test_path_replace(test262_handlers, handler_cls, expected): + tests_root, url_base = test262_handlers + _test_handler_path_replace(handler_cls, tests_root, url_base, expected) + + +@pytest.mark.parametrize("handler_cls, request_path, expected_metadata", [ + ( + T262WindowTestHandler, + "/test262/basic.test262-test.html", + [('script', '/third_party/test262/harness/assert.js'), ('script', '/third_party/test262/harness/sta.js')] + ), + ( + T262WindowTestHandler, + "/test262/negative.test262-test.html", + [('negative', 'TypeError')] + ), + ( + T262StrictWindowTestHandler, + "/test262/teststrict.test262-test.strict.html", + [('script', '/third_party/test262/harness/propertyHelper.js')] + ), +]) +def test_get_metadata(test262_handlers, handler_cls, request_path, expected_metadata): + tests_root, url_base = test262_handlers + _test_handler_get_metadata(handler_cls, tests_root, url_base, request_path, expected_metadata) + + +@pytest.mark.parametrize("handler_cls, request_path, expected_substrings", [ + # T262WindowHandler: Should contain the iframe pointing to the test + ( + T262WindowHandler, + "/test262/basic.test262.html", + [''] + ), + # T262WindowTestHandler: Should contain script tags + ( + T262WindowTestHandler, + "/test262/basic.test262-test.html", + ['', '', ''] + ), + # T262WindowModuleTestHandler: Should contain module import + ( + T262WindowModuleTestHandler, + "/test262/module.test262-module-test.html", + ['"] + ), + # Strict HTML Case: points to the .strict.js variant + ( + T262StrictWindowTestHandler, + "/test262/teststrict.test262-test.strict.html", + ['src="/test262/teststrict.test262.strict.js"'] + ), + # Strict JS Case: The handler that serves the actual script + ( + T262StrictHandler, + "/test262/teststrict.test262.strict.js", + ['"use strict";', "console.log('hello');"] + ), +]) +def test_wrapper_content(test262_handlers, handler_cls, request_path, expected_substrings): + tests_root, url_base = test262_handlers + _test_handler_wrapper_content(handler_cls, tests_root, url_base, request_path, expected_substrings) diff --git a/tools/wpt/requirements.txt b/tools/wpt/requirements.txt index ea6498ce15f509..f0220f9cc032f5 100644 --- a/tools/wpt/requirements.txt +++ b/tools/wpt/requirements.txt @@ -1,2 +1,4 @@ requests==2.32.3 types-requests==2.32.0.20241016 +pyyaml==6.0.1 +types-pyyaml==6.0.12.20241230 diff --git a/tools/wptrunner/wptrunner/browsers/chrome.py b/tools/wptrunner/wptrunner/browsers/chrome.py index 71e67811039b2e..c015a7d5ebdb82 100644 --- a/tools/wptrunner/wptrunner/browsers/chrome.py +++ b/tools/wptrunner/wptrunner/browsers/chrome.py @@ -33,7 +33,8 @@ "reftest": "ChromeDriverRefTestExecutor", "print-reftest": "ChromeDriverPrintRefTestExecutor", "wdspec": "WdspecExecutor", - "crashtest": "ChromeDriverCrashTestExecutor"}, + "crashtest": "ChromeDriverCrashTestExecutor", + "test262": "ChromeDriverTestharnessExecutor"}, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", "env_extras": "env_extras", diff --git a/tools/wptrunner/wptrunner/browsers/edge.py b/tools/wptrunner/wptrunner/browsers/edge.py index 82597c9312a4ad..6145916f626096 100644 --- a/tools/wptrunner/wptrunner/browsers/edge.py +++ b/tools/wptrunner/wptrunner/browsers/edge.py @@ -19,7 +19,8 @@ "reftest": "EdgeDriverRefTestExecutor", "print-reftest": "EdgeDriverPrintRefTestExecutor", "wdspec": "WdspecExecutor", - "crashtest": "WebDriverCrashtestExecutor"}, + "crashtest": "WebDriverCrashtestExecutor", + "test262": "EdgeDriverTestharnessExecutor"}, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", "env_extras": "env_extras", diff --git a/tools/wptrunner/wptrunner/browsers/firefox.py b/tools/wptrunner/wptrunner/browsers/firefox.py index 81e47197397c84..8bddda22106246 100644 --- a/tools/wptrunner/wptrunner/browsers/firefox.py +++ b/tools/wptrunner/wptrunner/browsers/firefox.py @@ -46,7 +46,8 @@ "testharness": "MarionetteTestharnessExecutor", "reftest": "MarionetteRefTestExecutor", "print-reftest": "MarionettePrintRefTestExecutor", - "wdspec": "MarionetteWdspecExecutor"}, + "wdspec": "MarionetteWdspecExecutor", + "test262": "MarionetteTestharnessExecutor"}, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", "env_extras": "env_extras", diff --git a/tools/wptrunner/wptrunner/browsers/safari.py b/tools/wptrunner/wptrunner/browsers/safari.py index fc3cc1138b31b6..e43d04b36ae209 100644 --- a/tools/wptrunner/wptrunner/browsers/safari.py +++ b/tools/wptrunner/wptrunner/browsers/safari.py @@ -22,7 +22,8 @@ "executor": {"testharness": "WebDriverTestharnessExecutor", "reftest": "WebDriverRefTestExecutor", "wdspec": "WdspecExecutor", - "crashtest": "WebDriverCrashtestExecutor"}, + "crashtag": "WebDriverCrashtestExecutor", + "test262": "WebDriverTestharnessExecutor"}, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", "env_extras": "env_extras", diff --git a/tools/wptrunner/wptrunner/browsers/servo.py b/tools/wptrunner/wptrunner/browsers/servo.py index 7a95b08e7ad9ef..96b841f00751ab 100644 --- a/tools/wptrunner/wptrunner/browsers/servo.py +++ b/tools/wptrunner/wptrunner/browsers/servo.py @@ -23,6 +23,7 @@ "testharness": "ServoTestharnessExecutor", "reftest": "ServoRefTestExecutor", "wdspec": "WdspecExecutor", + "test262": "ServoTestharnessExecutor", }, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", diff --git a/tools/wptrunner/wptrunner/browsers/webkit.py b/tools/wptrunner/wptrunner/browsers/webkit.py index a3e8d1361c3d69..183c685e64b62e 100644 --- a/tools/wptrunner/wptrunner/browsers/webkit.py +++ b/tools/wptrunner/wptrunner/browsers/webkit.py @@ -16,7 +16,8 @@ "executor": {"testharness": "WebDriverTestharnessExecutor", "reftest": "WebDriverRefTestExecutor", "wdspec": "WdspecExecutor", - "crashtest": "WebDriverCrashtestExecutor"}, + "crashtest": "WebDriverCrashtestExecutor", + "test262": "WebDriverTestharnessExecutor"}, "executor_kwargs": "executor_kwargs", "env_extras": "env_extras", "env_options": "env_options", diff --git a/tools/wptrunner/wptrunner/wpttest.py b/tools/wptrunner/wptrunner/wpttest.py index 9616f9c429f309..eb3c98f12ddca6 100644 --- a/tools/wptrunner/wptrunner/wpttest.py +++ b/tools/wptrunner/wptrunner/wpttest.py @@ -10,7 +10,7 @@ from .wptmanifest.parser import atoms atom_reset = atoms["Reset"] -enabled_tests = {"testharness", "reftest", "wdspec", "crashtest", "print-reftest"} +enabled_tests = {"testharness", "reftest", "wdspec", "crashtest", "print-reftest", "test262"} class Result(ABC): @@ -529,6 +529,10 @@ def id(self): return self.url +class Test262Test(TestharnessTest): + test_type = "test262" + + class ReftestTest(Test): """A reftest @@ -764,7 +768,8 @@ def from_manifest(cls, manifest_file, manifest_item, inherit_metadata, test_meta "print-reftest": PrintReftestTest, "testharness": TestharnessTest, "wdspec": WdspecTest, - "crashtest": CrashTest} + "crashtest": CrashTest, + "test262": Test262Test} def from_manifest(manifest_file, manifest_test, inherit_metadata, test_metadata):