Skip to content

Make tests more reproducible by seeding based on test paths#50

Merged
voltjia merged 1 commit intomasterfrom
make-tests-more-reproducible-by-seeding-based-on-test-paths
Oct 11, 2025
Merged

Make tests more reproducible by seeding based on test paths#50
voltjia merged 1 commit intomasterfrom
make-tests-more-reproducible-by-seeding-based-on-test-paths

Conversation

@voltjia
Copy link
Collaborator

@voltjia voltjia commented Oct 11, 2025

pytest output:

============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.4.1, pluggy-1.6.0
rootdir: /home/huang/ntops
configfile: pyproject.toml
plugins: jaxtyping-0.3.2, typeguard-4.4.4, cov-6.2.1, anyio-4.9.0
collected 636 items

tests/test_abs.py ........                                               [  1%]
tests/test_add.py ........                                               [  2%]
tests/test_addmm.py .F                                                   [  2%]
tests/test_bitwise_and.py ................                               [  5%]
tests/test_bitwise_not.py ................                               [  7%]
tests/test_bitwise_or.py ................                                [ 10%]
tests/test_bmm.py ..                                                     [ 10%]
tests/test_clamp.py ........                                             [ 11%]
tests/test_cos.py ........                                               [ 13%]
tests/test_div.py ........                                               [ 14%]
tests/test_dropout.py ........                                           [ 15%]
tests/test_eq.py ........                                                [ 16%]
tests/test_exp.py ........                                               [ 18%]
tests/test_ge.py ........                                                [ 19%]
tests/test_gelu.py ........                                              [ 20%]
tests/test_gt.py ........                                                [ 22%]
tests/test_isinf.py ........                                             [ 23%]
tests/test_isnan.py ........                                             [ 24%]
tests/test_layer_norm.py ............................................... [ 31%]
.................................................                        [ 39%]
tests/test_le.py ........                                                [ 40%]
tests/test_lt.py ........                                                [ 42%]
tests/test_mm.py ..                                                      [ 42%]
tests/test_mul.py ........                                               [ 43%]
tests/test_ne.py ........                                                [ 44%]
tests/test_neg.py ........                                               [ 46%]
tests/test_pow.py ........                                               [ 47%]
tests/test_relu.py ........                                              [ 48%]
tests/test_rms_norm.py ................................................. [ 56%]
...............                                                          [ 58%]
tests/test_rotary_position_embedding.py ................................ [ 63%]
........................................................................ [ 75%]
........................                                                 [ 78%]
tests/test_rsqrt.py ........                                             [ 80%]
tests/test_scaled_dot_product_attention.py ............................. [ 84%]
.................................................                        [ 92%]
tests/test_sigmoid.py ........                                           [ 93%]
tests/test_silu.py ........                                              [ 94%]
tests/test_sin.py ........                                               [ 96%]
tests/test_softmax.py ........                                           [ 97%]
tests/test_sub.py ........                                               [ 98%]
tests/test_tanh.py ........                                              [100%]

=================================== FAILURES ===================================
__________________ test_cuda[492-430-881-dtype1-0.025-0.025] ___________________

m = 492, n = 430, k = 881, dtype = torch.float16, atol = 0.025, rtol = 0.025

    @skip_if_cuda_not_available
    @pytest.mark.parametrize(*generate_arguments())
    def test_cuda(m, n, k, dtype, atol, rtol):
        device = "cuda"
    
        input = torch.randn((m, n), dtype=dtype, device=device)
        x = torch.randn((m, k), dtype=dtype, device=device)
        y = torch.randn((k, n), dtype=dtype, device=device)
        beta = gauss()
        alpha = gauss()
    
        ninetoothed_output = ntops.torch.addmm(input, x, y, beta=beta, alpha=alpha)
        reference_output = torch.addmm(input, x, y, beta=beta, alpha=alpha)
    
>       assert torch.allclose(ninetoothed_output, reference_output, atol=atol, rtol=rtol)
E       AssertionError: assert False
E        +  where False = <built-in method allclose of type object at 0x77fd92b7bfa0>(tensor([[ -17.7812,   16.5625,  -11.5156,  ...,   83.8750,   35.4375,\n           35.5625],\n        [  69.8750,   38.21...  7.2578,   91.1875,  -16.5469,  ..., -189.5000,  -89.9375,\n           28.7031]], device='cuda:0', dtype=torch.float16), tensor([[ -17.7812,   16.5625,  -11.5156,  ...,   83.9375,   35.4375,\n           35.5625],\n        [  69.9375,   38.21...  7.2461,   91.1875,  -16.5469,  ..., -189.6250,  -89.9375,\n           28.7188]], device='cuda:0', dtype=torch.float16), atol=0.025, rtol=0.025)
E        +    where <built-in method allclose of type object at 0x77fd92b7bfa0> = torch.allclose

tests/test_addmm.py:24: AssertionError
=========================== short test summary info ============================
FAILED tests/test_addmm.py::test_cuda[492-430-881-dtype1-0.025-0.025] - Asser...
================== 1 failed, 635 passed in 633.69s (0:10:33) ===================

@voltjia voltjia merged commit a8ed90a into master Oct 11, 2025
4 checks passed
@voltjia voltjia deleted the make-tests-more-reproducible-by-seeding-based-on-test-paths branch October 11, 2025 20:35
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

1 participant