fix(tier3): propagate Harbor dual-arm task suffixes and normalize canonical case ID resolution

CHANGELOG.md

@@ -6,6 +6,11 @@ All notable changes to SkillEvaluator are documented in this file.
 
 ### Fixed
 
+- Tier 3 Harbor dual-arm evaluation propagates arm suffixes (`-with-skill`,
+  `-without-skill`) to `[task] name` in staged native `task.toml` files,
+  normalizes external repository and namespace prefixes, and commutatively
+  resolves canonical case IDs across attempt and arm suffix combinations
+  while preserving expected case IDs.
 - Keep headings and comments inside fenced code examples in their enclosing Markdown
   section during Tier 2 content chunking, preserving original source line numbers.
 - Run the public Docker image as an unprivileged user, with writable default report and home directories.

src/skillevaluator/tier3/harbor/adapter.py

@@ -1794,6 +1794,7 @@ def _write_task_toml(
     pre_agent_setup: list[str] | None = None,
     task_resources: dict[str, int] | None = None,
     agent_workdir: str | None = None,
+    arm_suffix: str = "",
 ) -> None:
     entry_id = entry.get("id", "unknown")
     expected_skill = entry.get("expected_skill") or "none"
@@ -1803,16 +1804,19 @@ def _write_task_toml(
         raise TypeError("expected_skill must be a string before Harbor TOML serialization")
     if not isinstance(docker_image, str):
         raise TypeError("docker_image must be a string before Harbor TOML serialization")
+    if not isinstance(arm_suffix, str):
+        raise TypeError("arm_suffix must be a string before Harbor TOML serialization")
     docker_image_line = f"docker_image = {_toml_quote(docker_image)}\n" if docker_image else ""
     cpus = _task_resource_value(task_resources, "cpus", 2)
     memory_mb = _task_resource_value(task_resources, "memory_mb", 4096)
     storage_mb = _task_resource_value(task_resources, "storage_mb", 2048)
     workdir_line = f"workdir = {_toml_quote(agent_workdir)}\n" if agent_workdir else ""
 
+    task_name = f"nvidia/skillevaluator-{entry_id}{arm_suffix}"
     content = f"""schema_version = "1.3"
 
 [task]
-name = {_toml_quote(f"nvidia/skillevaluator-{entry_id}")}
+name = {_toml_quote(task_name)}
 description = {_toml_quote(f"Skill evaluation task for {expected_skill}")}
 
 [metadata]
@@ -4225,6 +4229,244 @@ def _native_entry_id(task_dir: Path) -> str:
     return task_dir.name
 
 
+_TOML_DOUBLE_QUOTE = '"'
+_TOML_SINGLE_QUOTE = "'"
+_TOML_MULTILINE_DOUBLE_QUOTE = '"""'
+_TOML_MULTILINE_SINGLE_QUOTE = "'''"
+_TOML_STRING_DELIMITERS = (
+    _TOML_MULTILINE_DOUBLE_QUOTE,
+    _TOML_MULTILINE_SINGLE_QUOTE,
+    _TOML_DOUBLE_QUOTE,
+    _TOML_SINGLE_QUOTE,
+)
+_TOML_ESCAPE_PAIR_LEN = 2  # backslash + escaped character
+
+
+def _skip_toml_string_literal(content: str, start: int) -> int:
+    """Return index immediately after the TOML string literal starting at *start*."""
+    n = len(content)
+    for delimiter in _TOML_STRING_DELIMITERS:
+        if not content.startswith(delimiter, start):
+            continue
+        delim_len = len(delimiter)
+        quote_char = delimiter[0]
+        supports_escapes = quote_char == _TOML_DOUBLE_QUOTE
+        is_multiline = delim_len > 1
+
+        i = start + delim_len
+        while i < n:
+            if supports_escapes and content[i] == "\\":
+                i += _TOML_ESCAPE_PAIR_LEN
+                continue
+            if content.startswith(delimiter, i):
+                i += delim_len
+                if is_multiline:
+                    while i < n and content[i] == quote_char:
+                        i += 1
+                return i
+            i += 1
+        return n
+    return start
+
+
+def _parse_toml_dotted_key(raw: str) -> tuple[str, ...]:
+    """Parse a TOML key or table path into unquoted segment names."""
+    parts: list[str] = []
+    for token in re.findall(r'"(?:\\.|[^"\\\r\n])*"|\'[^\'\r\n]*\'|[A-Za-z0-9_-]+', raw):
+        if (token.startswith('"') and token.endswith('"')) or (token.startswith("'") and token.endswith("'")):
+            parts.append(token[1:-1])
+        else:
+            parts.append(token)
+    return tuple(parts)
+
+
+def _find_toml_table_key_value_span(content: str, target_table: str, target_key: str) -> tuple[int, int] | None:
+    """Return the (start, end) character span of a string value for [target_table].target_key."""
+    n = len(content)
+    i = 0
+    current_table: tuple[str, ...] = ()
+    bracket_depth = 0
+
+    while i < n:
+        while i < n and content[i] in " \t":
+            i += 1
+        if i >= n:
+            break
+        if content[i] in "\r\n":
+            i += 1
+            continue
+        if content[i] == "#":
+            while i < n and content[i] != "\n":
+                i += 1
+            continue
+
+        if bracket_depth == 0 and content[i] == "[":
+            line_end = content.find("\n", i)
+            if line_end == -1:
+                line_end = n
+            line = content[i:line_end]
+            header_match = re.match(r"^\[(\[?)\s*([^\[\]#\r\n]+?)\s*\]\]?\s*(?:#.*)?\r?$", line)
+            if header_match:
+                is_array_table, raw_table = header_match.groups()
+                table_parts = _parse_toml_dotted_key(raw_table)
+                current_table = (f"[[{'.'.join(table_parts)}]]",) if is_array_table else table_parts
+                i = line_end + 1
+                continue
+
+        if bracket_depth == 0:
+            key_match = re.match(
+                r"""^(?:[A-Za-z0-9_-]+|"(?:\\.|[^"\\\r\n])*"|'[^'\r\n]*')(?:\s*\.\s*(?:[A-Za-z0-9_-]+|"(?:\\.|[^"\\\r\n])*"|'[^'\r\n]*'))*\s*=""",
+                content[i:],
+            )
+            if key_match:
+                raw_lhs = content[i : i + key_match.end() - 1].strip()
+                key_parts = _parse_toml_dotted_key(raw_lhs)
+                full_path = (*current_table, *key_parts)
+                val_start = i + key_match.end()
+                while val_start < n and content[val_start] in " \t":
+                    val_start += 1
+                if full_path == (target_table, target_key) and val_start < n and content[val_start] in "\"'":
+                    val_end = _skip_toml_string_literal(content, val_start)
+                    return val_start, val_end
+                i = val_start
+
+        while i < n:
+            ch = content[i]
+            if ch in "\"'":
+                i = _skip_toml_string_literal(content, i)
+                continue
+            if ch in "[({":
+                bracket_depth += 1
+                i += 1
+                continue
+            if ch in "])}":
+                bracket_depth = max(0, bracket_depth - 1)
+                i += 1
+                continue
+            if ch == "#":
+                while i < n and content[i] != "\n":
+                    i += 1
+                continue
+            if ch == "\n":
+                i += 1
+                if bracket_depth == 0:
+                    break
+                continue
+            i += 1
+
+    return None
+
+
+def _ensure_native_metadata_entry_id(content: str, entry_id: str) -> str:
+    """Ensure [metadata].entry_id is present in a native task.toml document."""
+    n = len(content)
+    i = 0
+    bracket_depth = 0
+    while i < n:
+        while i < n and content[i] in " \t":
+            i += 1
+        if i >= n:
+            break
+        if content[i] in "\r\n":
+            i += 1
+            continue
+        if content[i] == "#":
+            while i < n and content[i] != "\n":
+                i += 1
+            continue
+        if bracket_depth == 0 and content[i] == "[":
+            line_end = content.find("\n", i)
+            if line_end == -1:
+                line_end = n
+            line = content[i:line_end]
+            header_match = re.match(r"^\[(\[?)\s*([^\[\]#\r\n]+?)\s*\]\]?\s*(?:#.*)?\r?$", line)
+            if header_match:
+                is_array_table, raw_table = header_match.groups()
+                if not is_array_table and _parse_toml_dotted_key(raw_table) == ("metadata",):
+                    insert_pos = line_end + 1 if line_end < n else n
+                    prefix = content[:insert_pos]
+                    if not prefix.endswith("\n"):
+                        prefix += "\n"
+                    return f"{prefix}entry_id = {_toml_quote(entry_id)}\n{content[insert_pos:]}"
+                i = line_end + 1
+                continue
+        while i < n:
+            ch = content[i]
+            if ch in "\"'":
+                i = _skip_toml_string_literal(content, i)
+                continue
+            if ch in "[({":
+                bracket_depth += 1
+                i += 1
+                continue
+            if ch in "])}":
+                bracket_depth = max(0, bracket_depth - 1)
+                i += 1
+                continue
+            if ch == "#":
+                while i < n and content[i] != "\n":
+                    i += 1
+                continue
+            if ch == "\n":
+                i += 1
+                if bracket_depth == 0:
+                    break
+                continue
+            i += 1
+
+    suffix = "" if content.endswith("\n") else "\n"
+    return f"{content}{suffix}\n[metadata]\nentry_id = {_toml_quote(entry_id)}\n"
+
+
+def _append_native_task_name_suffix(
+    task_dir: Path,
+    arm_suffix: str,
+    *,
+    entry_id: str | None = None,
+) -> None:
+    """Append dual-arm suffix to [task] name in a native task's task.toml."""
+    if not arm_suffix:
+        return
+    task_toml = task_dir / "task.toml"
+    if not task_toml.exists():
+        return
+    try:
+        content = task_toml.read_text(encoding="utf-8")
+        data = tomllib.loads(content)
+    except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError):
+        return
+
+    task_table = data.get("task")
+    if not isinstance(task_table, dict):
+        return
+    old_name = task_table.get("name")
+    if not isinstance(old_name, str):
+        return
+
+    new_name = f"{old_name}{arm_suffix}"
+    span = _find_toml_table_key_value_span(content, "task", "name")
+    if span is None:
+        return
+    val_start, val_end = span
+    new_content = f"{content[:val_start]}{_toml_quote(new_name)}{content[val_end:]}"
+
+    metadata = data.get("metadata")
+    effective_entry_id = entry_id or (
+        str(metadata["entry_id"]) if isinstance(metadata, dict) and metadata.get("entry_id") else task_dir.name
+    )
+    if effective_entry_id and not (isinstance(metadata, dict) and "entry_id" in metadata):
+        candidate_content = _ensure_native_metadata_entry_id(new_content, effective_entry_id)
+        try:
+            tomllib.loads(candidate_content)
+            new_content = candidate_content
+        except tomllib.TOMLDecodeError:
+            pass
+
+    tomllib.loads(new_content)
+    if new_content != content:
+        task_toml.write_text(new_content, encoding="utf-8")
+
+
 def _environment_reference_names(value: object) -> set[str]:
     """Return portable shell-style environment references from a TOML value."""
     if not isinstance(value, str):
@@ -4579,12 +4821,15 @@ def _stage_native_harbor_tasks_into(
     task_resources: dict[str, int] | None = None,
     agent_workdir: str | None = None,
     baseline_aliases_prevalidated: bool = False,
+    arm_suffix: str = "",
 ) -> list[Path]:
     """Build native Harbor tasks inside a private, caller-owned directory.
 
     The source tree is copied first and all SkillEvaluator injections happen only in the
     staged result directory.
     """
+    if not isinstance(arm_suffix, str):
+        raise TypeError("arm_suffix must be a string before staging native Harbor tasks")
     _validate_runtime_discovery_env(runtime_env)
     _validate_runtime_loader_env(runtime_env)
     evals_dir = evaluator_skill_path / "evals"
@@ -4636,6 +4881,7 @@ def _stage_native_harbor_tasks_into(
         baseline_aliases_prevalidated = True
     for task_dir in task_dirs:
         entry_id = _native_entry_id(task_dir)
+        _append_native_task_name_suffix(task_dir, arm_suffix, entry_id=entry_id)
         native_agent_workdir = _native_task_workdir(task_dir)
         _ensure_native_skills_dir(task_dir)
         entry = entries_by_id.get(entry_id)
@@ -4767,9 +5013,13 @@ def stage_native_harbor_tasks(
     agent_workdir: str | None = None,
     evaluator_skill_path: Path | None = None,
     _baseline_alias_validation: _BaselineAliasValidation | None = None,
+    arm_suffix: str = "",
 ) -> list[Path]:
     """Stage native tasks privately, then publish one exact output snapshot."""
 
+    if not isinstance(arm_suffix, str):
+        raise TypeError("arm_suffix must be a string before staging native Harbor tasks")
+
     if evaluator_skill_path is None:
         with private_evaluator_skill_snapshot(skill_path, task_source="native_harbor") as private_skill_path:
             return stage_native_harbor_tasks(
@@ -4791,6 +5041,7 @@ def stage_native_harbor_tasks(
                 agent_workdir=agent_workdir,
                 evaluator_skill_path=private_skill_path,
                 _baseline_alias_validation=_baseline_alias_validation,
+                arm_suffix=arm_suffix,
             )
 
     baseline_aliases_prevalidated = False
@@ -4857,6 +5108,7 @@ def stage_native_harbor_tasks(
             task_resources=task_resources,
             agent_workdir=agent_workdir,
             baseline_aliases_prevalidated=baseline_aliases_prevalidated,
+            arm_suffix=arm_suffix,
         )
         relative_tasks = [task.relative_to(private_output) for task in private_tasks]
         if output_requires_provenance:
@@ -4915,6 +5167,7 @@ def _generate_harbor_tasks_into(
     task_resources: dict[str, int] | None = None,
     agent_workdir: str | None = None,
     baseline_aliases_prevalidated: bool = False,
+    arm_suffix: str = "",
 ) -> list[Path]:
     """Generate Harbor task directories inside a private output directory.
 
@@ -4947,6 +5200,8 @@ def _generate_harbor_tasks_into(
     Returns:
         List of generated task directory paths.
     """
+    if not isinstance(arm_suffix, str):
+        raise TypeError("arm_suffix must be a string before generating Harbor tasks")
     _validate_runtime_discovery_env(runtime_env)
     _validate_runtime_loader_env(runtime_env)
     agent_workdir = _validated_agent_workdir(agent_workdir)
@@ -5011,6 +5266,7 @@ def _generate_harbor_tasks_into(
             pre_agent_setup=pre_agent_setup,
             task_resources=task_resources,
             agent_workdir=agent_workdir,
+            arm_suffix=arm_suffix,
         )
         _copy_verifier(task_dir)
         custom_grader = _copy_custom_grader(task_dir, skill_path, grading_mode, evals_dir=evals_dir)
@@ -5446,9 +5702,13 @@ def generate_harbor_tasks(
     agent_workdir: str | None = None,
     evaluator_skill_path: Path | None = None,
     _baseline_alias_validation: _BaselineAliasValidation | None = None,
+    arm_suffix: str = "",
 ) -> list[Path]:
     """Generate tasks from one private evals snapshot, then publish exactly."""
 
+    if not isinstance(arm_suffix, str):
+        raise TypeError("arm_suffix must be a string before generating Harbor tasks")
+
     if evaluator_skill_path is None:
         if find_evals_file(skill_path) is None:
             raise FileNotFoundError(f"No evals dataset found in {skill_path / 'evals'}")
@@ -5472,6 +5732,7 @@ def generate_harbor_tasks(
                 agent_workdir=agent_workdir,
                 evaluator_skill_path=private_skill_path,
                 _baseline_alias_validation=_baseline_alias_validation,
+                arm_suffix=arm_suffix,
             )
     if find_evals_file(evaluator_skill_path) is None:
         raise FileNotFoundError(f"No evals dataset found in {evaluator_skill_path / 'evals'}")
@@ -5540,6 +5801,7 @@ def generate_harbor_tasks(
             task_resources=task_resources,
             agent_workdir=agent_workdir,
             baseline_aliases_prevalidated=baseline_aliases_prevalidated,
+            arm_suffix=arm_suffix,
         )
         relative_tasks = [task.relative_to(private_output) for task in private_tasks]
         if output_requires_provenance: