feat(manifest): Add initial tooling for Test262 test types [pt 1/5]

tools/manifest/commands.json

@@ -4,7 +4,10 @@
     "script": "run",
     "parser": "create_parser",
     "help": "Update the MANIFEST.json file",
-    "virtualenv": false
+    "virtualenv": true,
+    "requirements": [
+      "requirements.txt"
+    ]
   },
   "manifest-download": {
     "path": "download.py",

tools/manifest/item.py

@@ -198,6 +198,12 @@ def to_json(self) -> Tuple[Optional[Text], Dict[Text, Any]]:
         return rv
 
 
+class Test262Test(TestharnessTest):
+    __slots__ = ()
+
+    item_type = "test262"
+
+
 class RefTest(URLManifestItem):
     __slots__ = ("references",)
 

tools/manifest/manifest.py

@@ -17,6 +17,7 @@
                    SpecItem,
                    SupportFile,
                    TestharnessTest,
+                   Test262Test,
                    VisualTest,
                    WebDriverSpecTest)
 from .log import get_logger
@@ -49,7 +50,8 @@ class InvalidCacheError(Exception):
                                                 "conformancechecker": ConformanceCheckerTest,
                                                 "visual": VisualTest,
                                                 "spec": SpecItem,
-                                                "support": SupportFile}
+                                                "support": SupportFile,
+                                                "test262": Test262Test}
 
 
 def compute_manifest_items(source_file: SourceFile) -> Optional[Tuple[Tuple[Text, ...], Text, Set[ManifestItem], Text]]:

tools/manifest/sourcefile.py

@@ -24,11 +24,15 @@
                    RefTest,
                    SpecItem,
                    SupportFile,
+                   Test262Test,
                    TestharnessTest,
                    VisualTest,
                    WebDriverSpecTest)
+from .log import get_logger
 from .utils import cached_property
 
+from . import test262
+
 # Cannot do `from ..metadata.webfeatures.schema import WEB_FEATURES_YML_FILENAME`
 # because relative import beyond toplevel throws *ImportError*!
 from metadata.webfeatures.schema import WEB_FEATURES_YML_FILENAME  # type: ignore
@@ -421,6 +425,12 @@ def name_is_print_reftest(self) -> bool:
         return (self.markup_type is not None and
                 (self.type_flag == "print" or "print" in self.dir_path.split(os.path.sep)))
 
+    @property
+    def name_is_test262(self) -> bool:
+        """Check if the file name matches the conditions for the file to be a
+        test262 file"""
+        return ("test262" in self.dir_path.split(os.path.sep) and self.ext == ".js")
+
     @property
     def markup_type(self) -> Optional[Text]:
         """Return the type of markup contained in a file, based on its extension,
@@ -470,12 +480,32 @@ def pac_nodes(self) -> List[ElementTree.Element]:
         assert self.root is not None
         return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='pac']")
 
+    @cached_property
+    def test262_test_record(self) -> Optional[test262.TestRecord]:
+        if self.name_is_test262:
+            with self.open() as f:
+                return test262.parse(get_logger(), f.read().decode('utf-8'), self.path)
+        else:
+            return None
+
     @cached_property
     def script_metadata(self) -> Optional[List[Tuple[Text, Text]]]:
         if self.name_is_worker or self.name_is_multi_global or self.name_is_window or self.name_is_extension:
             regexp = js_meta_re
         elif self.name_is_webdriver:
             regexp = python_meta_re
+        elif self.name_is_test262:
+            if self.test262_test_record is None:
+                return None
+            paths: List[Tuple[Text, Text]] = []
+            if self.test262_test_record.includes is None:
+                return paths
+            for filename in self.test262_test_record.includes:
+                if filename in ("assert.js", "sta.js"):
+                    paths.append(('script', "/third_party/test262/harness/%s" % filename))
+                else:
+                    paths.append(('script', "/resources/test262/%s" % filename))
+            return paths
         else:
             return None
 
@@ -917,6 +947,9 @@ def possible_types(self) -> Set[Text]:
         if self.name_is_window:
             return {TestharnessTest.item_type}
 
+        if self.name_is_test262:
+            return {Test262Test.item_type, SupportFile.item_type}
+
         if self.name_is_extension:
             return {TestharnessTest.item_type}
 
@@ -1100,6 +1133,38 @@ def manifest_items(self) -> Tuple[Text, List[ManifestItem]]:
             ]
             rv = TestharnessTest.item_type, tests
 
+        elif self.name_is_test262:
+            if self.test262_test_record is None:
+                rv = "support", [
+                    SupportFile(
+                        self.tests_root,
+                        self.rel_path
+                    )]
+            else:
+                suffix = ".test262"
+                if self.test262_test_record.is_module:
+                    suffix += "-module"
+                elif self.test262_test_record.is_only_strict:
+                    # Modules are always strict mode, so only append strict for
+                    # non-module tests.
+                    suffix += ".strict"
+                suffix += ".html"
+                test_url = replace_end(self.rel_url, ".js", suffix)
+                tests = [
+                    Test262Test(
+                        self.tests_root,
+                        self.rel_path,
+                        self.url_base,
+                        test_url + variant,
+                        timeout=self.timeout,
+                        pac=self.pac,
+                        testdriver_features=self.testdriver_features,
+                        script_metadata=self.script_metadata
+                    )
+                    for variant in self.test_variants
+                ]
+                rv = Test262Test.item_type, tests
+
         elif self.content_is_css_manual and not self.name_is_reference:
             rv = ManualTest.item_type, [
                 ManualTest(

tools/manifest/test262.py

@@ -0,0 +1,73 @@
+from __future__ import print_function
+
+from dataclasses import dataclass
+from logging import Logger
+from typing import Dict, List, Optional, Text, Tuple
+
+import re
+
+# Matches trailing whitespace and any following blank lines.
+_BLANK_LINES = r"([ \t]*[\r\n]{1,2})*"
+
+# Matches the YAML frontmatter block.
+_YAML_PATTERN = re.compile(r"/\*---(.*)---\*/" + _BLANK_LINES, re.DOTALL)
+
+_STRIP_CONTROL_CHARS = re.compile(r'[\x7f-\x9f]')
+
+
+
+@dataclass
+class TestRecord:
+    test: str
+    includes: Optional[List[Text]] = None
+    negative: Optional[Dict[Text, Text]] = None
+    is_only_strict: bool = False
+    is_module: bool = False
+
+def _yaml_attr_parser(logger: Logger, test_record: TestRecord, attrs: Text, name: Text) -> None:
+    import yaml
+    parsed = yaml.safe_load(re.sub(_STRIP_CONTROL_CHARS, ' ', attrs))
+    if parsed is None:
+        logger.error("Failed to parse yaml in name %s" % name)
+        return
+
+    for key, value in parsed.items():
+        if key == "negative":
+            test_record.negative = value
+        elif key == "flags":
+            if isinstance(value, list):
+                for flag in value:
+                    if flag == "onlyStrict":
+                        test_record.is_only_strict = True
+                    elif flag == "module":
+                        test_record.is_module = True
+        elif key == "includes":
+            test_record.includes = value
+
+
+def _find_attrs(src: Text) -> Tuple[Optional[Text], Optional[Text]]:
+    match = _YAML_PATTERN.search(src)
+    if not match:
+        return (None, None)
+
+    return (match.group(0), match.group(1).strip())
+
+
+def parse(logger: Logger, src: Text, name: Text) -> Optional[TestRecord]:
+    if name.endswith('_FIXTURE.js'):
+        return None
+
+    # Find the YAML frontmatter.
+    (frontmatter, attrs) = _find_attrs(src)
+
+    # YAML frontmatter is required for all tests.
+    if frontmatter is None:
+        logger.error("Missing frontmatter: %s" % name)
+        return None
+
+    test_record = TestRecord(test = src)
+
+    if attrs:
+        _yaml_attr_parser(logger, test_record, attrs, name)
+
+    return test_record

tools/manifest/tests/test_sourcefile.py

@@ -1013,3 +1013,39 @@ def test_html_testdriver_features(features):
 
     s = create("html/test.html", contents=contents)
     assert s.testdriver_features == features
+
+@pytest.mark.parametrize("rel_path, is_test262", [
+    ("test262/test.js", True),
+    ("other/test.js", False),
+])
+def test_name_is_test262(rel_path, is_test262):
+    tests_root = "/tmp"
+    url_base = "/"
+    sf = SourceFile(tests_root, rel_path, url_base)
+    assert sf.name_is_test262 == is_test262
+
+def test_test262_test_record():
+    contents = b"""/*---
+description: A simple test
+---*/"""
+    sf = create("test262/test.js", contents=contents)
+    record = sf.test262_test_record
+    assert record is not None
+
+@pytest.mark.parametrize("rel_path, contents, expected_url", [
+    ("test262/test.js",
+     b"/*---\ndescription: A simple test\n---*/",
+     "/test262/test.test262.html"),
+    ("test262/module.js",
+     b"/*---\ndescription: A module test\nflags: [module]\n---*/",
+     "/test262/module.test262-module.html"),
+    ("test262/strict.js",
+     b"/*---\ndescription: A strict mode test\nflags: [onlyStrict]\n---*/",
+     "/test262/strict.test262.strict.html"),
+])
+def test_manifest_items_test262(rel_path, contents, expected_url):
+    sf = create(rel_path, contents=contents)
+    item_type, items = sf.manifest_items()
+    assert item_type == "test262"
+    assert len(items) == 1
+    assert items[0].url == expected_url

tools/manifest/tests/test_test262.py

@@ -0,0 +1,108 @@
+# mypy: allow-untyped-defs
+
+import pytest
+
+from tools.manifest.log import get_logger
+from tools.manifest.test262 import parse, TestRecord
+
+TestRecord.__test__ = False  # type: ignore[attr-defined]
+
+@pytest.mark.parametrize("name, src, expected_record", [
+    (
+        "test.js",
+        """/*---
+description: A simple test
+features: [Test262]
+---*/
+assert.sameValue(1, 1);
+""",
+        TestRecord("""/*---
+description: A simple test
+features: [Test262]
+---*/
+assert.sameValue(1, 1);
+""", includes=None, negative=None, is_module=False, is_only_strict=False)
+    ),
+    (
+        "no_frontmatter.js",
+        """assert.sameValue(1, 1);""",
+        None
+    ),
+    (
+        "test_FIXTURE.js",
+        """/*---
+description: A fixture file
+---*/
+assert.sameValue(1, 1);
+""",
+        None
+    ),
+    (
+        "flags-module.js",
+        """/*---
+description: Test with module flag
+flags: [raw, module]
+---*/
+assert.sameValue(1, 1);
+""",
+        TestRecord("""/*---
+description: Test with module flag
+flags: [raw, module]
+---*/
+assert.sameValue(1, 1);
+""", includes=None, negative=None, is_module=True, is_only_strict=False)
+    ),
+    (
+        "flags-onlyStrict.js",
+        """/*---
+description: Test with onlyStrict flag
+flags: [raw, onlyStrict]
+---*/
+assert.sameValue(1, 1);
+""",
+        TestRecord("""/*---
+description: Test with onlyStrict flag
+flags: [raw, onlyStrict]
+---*/
+assert.sameValue(1, 1);
+""", includes=None, negative=None, is_module=False, is_only_strict=True)
+    ),
+    (
+        "negative.js",
+        """/*---
+description: Negative test
+negative:
+  phase: runtime
+  type: TypeError
+---*/
+throw new TypeError();
+""",
+        TestRecord("""/*---
+description: Negative test
+negative:
+  phase: runtime
+  type: TypeError
+---*/
+throw new TypeError();
+""", includes=None, negative={"phase": "runtime", "type": "TypeError"}, is_module=False, is_only_strict=False)
+    ),
+    (
+        "includes.js",
+        """/*---
+description: Test with includes
+includes: [assert.js, sta.js]
+---*/
+assert.sameValue(1, 1);
+""",
+        TestRecord("""/*---
+description: Test with includes
+includes: [assert.js, sta.js]
+---*/
+assert.sameValue(1, 1);
+""", includes=["assert.js", "sta.js"], negative=None, is_module=False, is_only_strict=False)
+    ),
+])
+def test_test262_parser(name, src, expected_record):
+    record = parse(get_logger(), src, name)
+
+    assert expected_record == record

tools/wpt/requirements.txt

@@ -1,2 +1,4 @@
 requests==2.32.3
 types-requests==2.32.0.20241016
+pyyaml==6.0.1
+types-pyyaml==6.0.12.20241230