NVIDIA-NeMo
diff --git a/‎resources_servers/code_gen/lcb_integration/testing_util.py‎
Lines changed: 80 additions & 12 deletions b/‎resources_servers/code_gen/lcb_integration/testing_util.py‎
Lines changed: 80 additions & 12 deletions
diff --git a/‎resources_servers/code_gen/tests/test_float_tolerance.py‎
Lines changed: 68 additions & 0 deletions b/‎resources_servers/code_gen/tests/test_float_tolerance.py‎
Lines changed: 68 additions & 0 deletions
diff --git a/‎resources_servers/code_gen/tests/test_grader_environment.py‎
Lines changed: 155 additions & 0 deletions b/‎resources_servers/code_gen/tests/test_grader_environment.py‎
Lines changed: 155 additions & 0 deletions
@@ -27,7 +27,7 @@
 from datetime import datetime
 from decimal import Decimal
 from enum import Enum
-from io import BytesIO, StringIO
+from io import BytesIO, StringIO, TextIOWrapper
 
 # from pyext import RuntimeModule
 from types import ModuleType
@@ -38,7 +38,7 @@
 import numpy as np
 
 
-import_string = "from string import *\nfrom re import *\nfrom datetime import *\nfrom collections import *\nfrom heapq import *\nfrom bisect import *\nfrom copy import *\nfrom math import *\nfrom random import *\nfrom statistics import *\nfrom itertools import *\nfrom functools import *\nfrom operator import *\nfrom io import *\nfrom sys import *\nfrom json import *\nfrom builtins import *\nfrom typing import *\nimport string\nimport re\nimport datetime\nimport collections\nimport heapq\nimport bisect\nimport copy\nimport math\nimport random\nimport statistics\nimport itertools\nimport functools\nimport operator\nimport io\nimport sys\nimport json\nsys.setrecursionlimit(50000)\n"
+import_string = "from string import *\nfrom re import *\nfrom datetime import *\nfrom collections import *\nfrom heapq import *\nfrom bisect import *\nfrom copy import *\nfrom math import *\nfrom random import *\nfrom statistics import *\nfrom itertools import *\nfrom functools import *\nfrom operator import *\nfrom io import *\nfrom sys import *\nfrom json import *\nfrom builtins import *\nfrom typing import *\nimport string\nimport re\nimport datetime\nimport collections\nimport heapq\nimport bisect\nimport copy\nimport math\nimport random\nimport statistics\nimport itertools\nimport functools\nimport operator\nimport io\nimport sys\nimport json\nsys.setrecursionlimit(50000)\nsys.set_int_max_str_digits(1 << 20)\n"
 
 
 def truncatefn(s, length=300):
@@ -77,15 +77,24 @@ def timeout_handler(signum, frame):
 class Capturing(list):
     def __enter__(self):
         self._stdout = sys.stdout
-        sys.stdout = self._stringio = StringIO()
-        # Make closing the StringIO a no-op
-        self._stringio.close = lambda x: 1
+        self._bytes_buffer = BytesIO()
+        self._stringio = TextIOWrapper(
+            self._bytes_buffer,
+            encoding="utf-8",
+            newline="",
+            write_through=True,
+        )
+        # Make closing the stream a no-op
+        self._stringio.close = lambda *args, **kwargs: None
+        sys.stdout = self._stringio
         return self
 
     def __exit__(self, *args):
-        self.append(self._stringio.getvalue())
-        del self._stringio  # free up some memory
+        self._stringio.flush()
+        self.append(self._bytes_buffer.getvalue().decode("utf-8", errors="replace"))
         sys.stdout = self._stdout
+        del self._stringio  # free up some memory
+        del self._bytes_buffer
 
 
 # Custom mock for sys.stdin that supports buffer attribute
@@ -253,13 +262,63 @@ def convert_line_to_decimals(line: str) -> tuple[bool, list[Decimal]]:
     return True, decimal_line
 
 
+FLOAT_ERROR_TOLERANCE = Decimal("1e-6")
+
+
+def token_is_floating_point(token: str) -> bool:
+    return "." in token or "e" in token.lower()
+
+
+def decimal_tokens_match(expected_token: str, predicted_token: str, expected: Decimal, predicted: Decimal) -> bool:
+    if expected == predicted:
+        return True
+    if not (token_is_floating_point(expected_token) or token_is_floating_point(predicted_token)):
+        return False
+    if not (expected.is_finite() and predicted.is_finite()):
+        return False
+    absolute_error = abs(expected - predicted)
+    relative_scale = max(abs(expected), Decimal(1))
+    return absolute_error <= FLOAT_ERROR_TOLERANCE or absolute_error / relative_scale <= FLOAT_ERROR_TOLERANCE
+
+
+def decimal_lines_match(
+    expected_tokens: list[str],
+    predicted_tokens: list[str],
+    expected_line: list[Decimal],
+    predicted_line: list[Decimal],
+) -> bool:
+    if len(expected_line) != len(predicted_line):
+        return False
+    return all(
+        decimal_tokens_match(expected_token, predicted_token, expected, predicted)
+        for expected_token, predicted_token, expected, predicted in zip(
+            expected_tokens, predicted_tokens, expected_line, predicted_line
+        )
+    )
+
+
 def get_stripped_lines(val: str):
     ## you don't want empty lines to add empty list after splitlines!
     val = val.strip()
 
     return [val_line.strip() for val_line in val.split("\n")]
 
 
+def values_are_close(expected, predicted) -> bool:
+    if expected == predicted:
+        return True
+    if isinstance(expected, bool) or isinstance(predicted, bool):
+        return False
+    if not (isinstance(expected, float) or isinstance(predicted, float)):
+        return False
+    if not isinstance(expected, (int, float, str)) or not isinstance(predicted, (int, float, str)):
+        return False
+    try:
+        return bool(np.allclose(float(predicted), float(expected)))
+    except (TypeError, ValueError):
+        return False
+
+
 def grade_call_based(code: str, all_inputs: list, all_outputs: list, fn_name: str, timeout: int):
     # call-based clean up logic
     # need to wrap in try-catch logic after to catch the correct errors, but for now this is fine.
@@ -304,10 +363,14 @@ def grade_call_based(code: str, all_inputs: list, all_outputs: list, fn_name: st
             except Exception:
                 pass
 
-            try:
-                tmp_result = tmp_result or (np.allclose(float(prediction), float(gt_out)))
-            except Exception:
-                pass
+            if not tmp_result:
+                tmp_result = values_are_close(gt_out, prediction)
+
+            if not tmp_result and isinstance(prediction, list) and isinstance(gt_out, list):
+                tmp_result = len(prediction) == len(gt_out) and all(
+                    values_are_close(expected_item, predicted_item)
+                    for predicted_item, expected_item in zip(prediction, gt_out)
+                )
 
             all_results.append(tmp_result)
 
@@ -455,7 +518,12 @@ def grade_stdio(
                 all_results.append(-2)
                 return all_results, WA_send_args
 
-            if decimal_prediction_line == decimal_gtout_line:
+            if decimal_lines_match(
+                stripped_gt_out_line.split(),
+                stripped_prediction_line.split(),
+                decimal_gtout_line,
+                decimal_prediction_line,
+            ):
                 continue
 
             all_results.append(-2)
 
@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for floating-point tolerance in stdout answer comparison."""
+
+from __future__ import annotations
+
+from lcb_integration.testing_util import convert_line_to_decimals, decimal_lines_match
+
+
+def _match(expected: str, predicted: str) -> bool:
+    expected_ok, expected_line = convert_line_to_decimals(expected)
+    predicted_ok, predicted_line = convert_line_to_decimals(predicted)
+    assert expected_ok and predicted_ok
+    return decimal_lines_match(expected.split(), predicted.split(), expected_line, predicted_line)
+
+
+class TestFloatsGetTolerance:
+    """Judges accept 1e-6 absolute or relative error on floating-point output."""
+
+    def test_exact_float_matches(self):
+        assert _match("0.333333333333", "0.333333333333")
+
+    def test_small_relative_error_matches(self):
+        assert _match("0.333333333333", "0.3333333333")
+
+    def test_six_decimal_places_matches_twelve(self):
+        assert _match("0.333333333333", "0.333333")
+
+    def test_relative_error_on_large_magnitude_matches(self):
+        assert _match("1000000.0", "1000000.0000001")
+
+    def test_scientific_notation_matches(self):
+        assert _match("1e-7", "0.0000001")
+
+    def test_error_above_tolerance_fails(self):
+        assert not _match("0.333333333333", "0.334")
+
+    def test_every_token_must_be_within_tolerance(self):
+        assert _match("1.0 3.0 3.5", "1.0000001 2.9999999 3.5")
+        assert not _match("1.0 3.0 3.5", "1.0000001 2.5 3.5")
+
+
+class TestIntegersStayExact:
+    """Tolerance must not resurrect the np.isclose large-integer false positive."""
+
+    def test_large_integer_off_by_one_fails(self):
+        assert not _match("50000000000000000", "50000000000000001")
+
+    def test_small_integer_off_by_one_fails(self):
+        assert not _match("41", "42")
+
+    def test_equal_integers_match(self):
+        assert _match("50000000000000000", "50000000000000000")
+
+
+class TestStructuralMismatches:
+    def test_token_count_mismatch_fails(self):
+        assert not _match("1.0", "1.0 2.0")
+
+    def test_non_finite_does_not_crash(self):
+        assert not _match("1.0", "inf")
+        assert not _match("1.0", "nan")
@@ -0,0 +1,155 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for the environment submitted code is graded in: stdout capture and interpreter limits."""
+
+from __future__ import annotations
+
+import sys
+
+import pytest
+from lcb_integration.testing_util import (
+    Capturing,
+    grade_call_based,
+    import_string,
+    values_are_close,
+)
+
+
+class TestStdoutCaptureSupportsBuffer:
+    """Submitted code may use the `sys.stdout.buffer.write` fast-output idiom."""
+
+    def test_buffer_write_is_captured(self):
+        with Capturing() as captured:
+            sys.stdout.buffer.write(b"42\n")
+        assert captured[0] == "42\n"
+
+    def test_text_writes_are_captured(self):
+        with Capturing() as captured:
+            print("hello")
+        assert captured[0] == "hello\n"
+
+    def test_text_and_buffer_writes_keep_order(self):
+        with Capturing() as captured:
+            print("a")
+            sys.stdout.buffer.write(b"b\n")
+            print("c")
+        assert captured[0] == "a\nb\nc\n"
+
+    def test_closing_stdout_does_not_lose_output(self):
+        with Capturing() as captured:
+            print("before")
+            sys.stdout.close()
+            print("after")
+        assert captured[0] == "before\nafter\n"
+
+    def test_stdout_is_restored(self):
+        original = sys.stdout
+        with Capturing():
+            print("ignored")
+        assert sys.stdout is original
+
+
+class TestInterpreterLimits:
+    """Competitive answers can be very large integers, which Python refuses to print by default."""
+
+    def test_import_string_raises_the_int_to_str_limit(self):
+        previous = sys.get_int_max_str_digits()
+        try:
+            exec(import_string, {})
+            assert sys.get_int_max_str_digits() >= 1 << 20
+            assert len(str(10**10000)) == 10001
+        finally:
+            sys.set_int_max_str_digits(previous)
+
+    def test_default_limit_would_reject_a_large_answer(self):
+        previous = sys.get_int_max_str_digits()
+        try:
+            sys.set_int_max_str_digits(4300)
+            with pytest.raises(ValueError):
+                str(10**10000)
+        finally:
+            sys.set_int_max_str_digits(previous)
+
+
+class TestCallBasedFloatLists:
+    """A returned list of floats within tolerance must not be scored Wrong Answer."""
+
+    def test_float_list_within_tolerance_passes(self):
+        code = "class Solution:\n    def f(self, n):\n        return [0.1000000001, 0.2]\n"
+        results, _metadata = grade_call_based(
+            code=code, all_inputs=["[1]"], all_outputs=["[0.1, 0.2]"], fn_name="f", timeout=10
+        )
+        assert results == [True]
+
+    def test_float_list_outside_tolerance_fails(self):
+        code = "class Solution:\n    def f(self, n):\n        return [0.5, 0.2]\n"
+        results, _metadata = grade_call_based(
+            code=code, all_inputs=["[1]"], all_outputs=["[0.1, 0.2]"], fn_name="f", timeout=10
+        )
+        assert results != [True]
+
+    def test_float_list_of_different_length_fails(self):
+        code = "class Solution:\n    def f(self, n):\n        return [0.1, 0.2, 0.3]\n"
+        results, _metadata = grade_call_based(
+            code=code, all_inputs=["[1]"], all_outputs=["[0.1, 0.2]"], fn_name="f", timeout=10
+        )
+        assert results != [True]
+
+    def test_exact_int_list_still_passes(self):
+        code = "class Solution:\n    def f(self, n):\n        return [1, 2, 3]\n"
+        results, _metadata = grade_call_based(
+            code=code, all_inputs=["[1]"], all_outputs=["[1, 2, 3]"], fn_name="f", timeout=10
+        )
+        assert results == [True]
+
+
+class TestCallBasedIntegersStayExact:
+    """Tolerance must never apply to integers, in either the scalar or the list case."""
+
+    def test_large_int_list_off_by_one_fails(self):
+        code = "class Solution:\n    def f(self, n):\n        return [50000000000000001]\n"
+        results, _metadata = grade_call_based(
+            code=code,
+            all_inputs=["[1]"],
+            all_outputs=["[50000000000000000]"],
+            fn_name="f",
+            timeout=10,
+        )
+        assert results != [True]
+
+    def test_large_scalar_int_off_by_one_fails(self):
+        code = "class Solution:\n    def f(self, n):\n        return 50000000000000001\n"
+        results, _metadata = grade_call_based(
+            code=code,
+            all_inputs=["[1]"],
+            all_outputs=["50000000000000000"],
+            fn_name="f",
+            timeout=10,
+        )
+        assert results != [True]
+
+    def test_small_int_list_off_by_one_fails(self):
+        code = "class Solution:\n    def f(self, n):\n        return [1, 2, 4]\n"
+        results, _metadata = grade_call_based(
+            code=code, all_inputs=["[1]"], all_outputs=["[1, 2, 3]"], fn_name="f", timeout=10
+        )
+        assert results != [True]
+
+    def test_values_are_close_gates_on_float_type(self):
+        assert not values_are_close(50000000000000000, 50000000000000001)
+        assert not values_are_close(41, 42)
+        assert values_are_close(50000000000000000, 50000000000000000)
+        assert values_are_close(0.1, 0.1000000001)
+        assert not values_are_close(0.1, 0.5)
+
+    def test_booleans_never_get_tolerance(self):
+        assert not values_are_close(True, 0.9999999999)
+        assert not values_are_close(1.0, False)
+        assert values_are_close(True, True)
+        assert values_are_close(1.0, True)