Skip to content

Commit 6ba2548

Browse files
committed
Test
1 parent 3aada3f commit 6ba2548

File tree

5 files changed

+263
-0
lines changed

5 files changed

+263
-0
lines changed

.ci/scripts/wheel/test_wheel.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
"""
9+
Cross-platform wheel smoke-test entry point.
10+
11+
The PyTorch Tokenizers wheel should already be installed in the active
12+
environment when this script runs. We execute the Python unit test suite
13+
to ensure the wheel exposes the expected bindings and core functionality.
14+
"""
15+
16+
from __future__ import annotations
17+
18+
import os
19+
import subprocess
20+
import sys
21+
from pathlib import Path
22+
from typing import Sequence
23+
24+
25+
def run_pytest(test_files: Sequence[Path], cwd: Path) -> int:
26+
"""Execute pytest on the provided files and propagate the return code."""
27+
env = os.environ.copy()
28+
pythonpath_entries = [str(cwd)]
29+
if existing_path := env.get("PYTHONPATH"):
30+
pythonpath_entries.append(existing_path)
31+
env["PYTHONPATH"] = os.pathsep.join(pythonpath_entries)
32+
33+
cmd = [
34+
sys.executable,
35+
"-m",
36+
"pytest",
37+
"-vv",
38+
*(str(test) for test in test_files),
39+
]
40+
41+
print(f"Running pytest with: {' '.join(cmd)}")
42+
result = subprocess.run(cmd, cwd=str(cwd), env=env, check=False)
43+
return result.returncode
44+
45+
46+
def main() -> int:
47+
repo_root = Path(__file__).resolve().parents[3]
48+
test_dir = repo_root / "test"
49+
50+
if not test_dir.exists():
51+
print(f"ERROR: Test directory not found: {test_dir}", file=sys.stderr)
52+
return 1
53+
54+
return run_pytest([test_dir], repo_root)
55+
56+
57+
if __name__ == "__main__":
58+
sys.exit(main())
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Nova Build Wheels (Linux AArch64)
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- .ci/**/*
8+
- .github/workflows/build-wheels-linux-aarch64.yml
9+
- pyproject.toml
10+
- setup.py
11+
tags:
12+
- ciflow/binaries/*
13+
push:
14+
tags:
15+
- release/*
16+
tags:
17+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
18+
# Release candidate tags look like: v1.11.0-rc1
19+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
20+
- ciflow/binaries/*
21+
release:
22+
types: [published]
23+
24+
jobs:
25+
generate-matrix:
26+
name: Generate Linux AArch64 matrix
27+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
28+
with:
29+
package-type: wheel
30+
os: linux-aarch64
31+
test-infra-repository: pytorch/test-infra
32+
test-infra-ref: main
33+
with-cuda: disable
34+
with-rocm: disable
35+
with-cpu: enable
36+
with-xpu: disable
37+
python-versions: ${{ env.PYTHON_VERSIONS_JSON }}
38+
39+
build:
40+
name: Build Linux AArch64 wheels
41+
needs: generate-matrix
42+
uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
43+
with:
44+
repository: meta-pytorch/tokenizers
45+
ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }}
46+
test-infra-repository: pytorch/test-infra
47+
test-infra-ref: main
48+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
49+
trigger-event: ${{ github.event_name }}
50+
package-name: pytorch_tokenizers
51+
smoke-test-script: .ci/scripts/wheel/test_wheel.py
52+
architecture: aarch64
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: Nova Build Wheels (Linux x86_64)
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- .ci/**/*
8+
- .github/workflows/build-wheels-windows.yml
9+
- pyproject.toml
10+
- setup.py
11+
tags:
12+
- ciflow/binaries/*
13+
push:
14+
tags:
15+
- release/*
16+
tags:
17+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
18+
# Release candidate tags look like: v1.11.0-rc1
19+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
20+
- ciflow/binaries/*
21+
release:
22+
types: [published]
23+
24+
jobs:
25+
generate-matrix:
26+
name: Generate Linux matrix
27+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
28+
with:
29+
package-type: wheel
30+
os: linux
31+
test-infra-repository: pytorch/test-infra
32+
test-infra-ref: main
33+
with-cuda: disable
34+
with-rocm: disable
35+
with-cpu: enable
36+
with-xpu: disable
37+
python-versions: ${{ env.PYTHON_VERSIONS_JSON }}
38+
39+
build:
40+
name: Build Linux wheels
41+
needs: generate-matrix
42+
uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
43+
with:
44+
repository: meta-pytorch/tokenizers
45+
ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }}
46+
test-infra-repository: pytorch/test-infra
47+
test-infra-ref: main
48+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
49+
trigger-event: ${{ github.event_name }}
50+
package-name: pytorch_tokenizers
51+
smoke-test-script: .ci/scripts/wheel/test_wheel.py
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: Nova Build Wheels (macOS arm64)
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- .ci/**/*
8+
- .github/workflows/build-wheels-macos-arm64.yml
9+
- pyproject.toml
10+
- setup.py
11+
tags:
12+
- ciflow/binaries/*
13+
push:
14+
tags:
15+
- release/*
16+
tags:
17+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
18+
# Release candidate tags look like: v1.11.0-rc1
19+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
20+
- ciflow/binaries/*
21+
release:
22+
types: [published]
23+
24+
jobs:
25+
generate-matrix:
26+
name: Generate macOS matrix
27+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
28+
with:
29+
package-type: wheel
30+
os: macos-arm64
31+
test-infra-repository: pytorch/test-infra
32+
test-infra-ref: main
33+
with-cuda: disable
34+
with-rocm: disable
35+
with-cpu: enable
36+
with-xpu: disable
37+
python-versions: ${{ env.PYTHON_VERSIONS_JSON }}
38+
39+
build:
40+
name: Build macOS wheels
41+
needs: generate-matrix
42+
uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main
43+
with:
44+
repository: meta-pytorch/tokenizers
45+
ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }}
46+
test-infra-repository: pytorch/test-infra
47+
test-infra-ref: main
48+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
49+
trigger-event: ${{ github.event_name }}
50+
package-name: pytorch_tokenizers
51+
smoke-test-script: .ci/scripts/wheel/test_wheel.py
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: Nova Build Wheels (Windows)
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- .ci/**/*
8+
- .github/workflows/build-wheels-windows.yml
9+
- pyproject.toml
10+
- setup.py
11+
tags:
12+
- ciflow/binaries/*
13+
push:
14+
tags:
15+
- release/*
16+
tags:
17+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
18+
# Release candidate tags look like: v1.11.0-rc1
19+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
20+
- ciflow/binaries/*
21+
release:
22+
types: [published]
23+
24+
jobs:
25+
generate-matrix:
26+
name: Generate Windows matrix
27+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
28+
with:
29+
package-type: wheel
30+
os: windows
31+
test-infra-repository: pytorch/test-infra
32+
test-infra-ref: main
33+
with-cuda: disable
34+
with-rocm: disable
35+
with-cpu: enable
36+
with-xpu: disable
37+
python-versions: ${{ env.PYTHON_VERSIONS_JSON }}
38+
39+
build:
40+
name: Build Windows wheels
41+
needs: generate-matrix
42+
uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main
43+
with:
44+
repository: meta-pytorch/tokenizers
45+
ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }}
46+
test-infra-repository: pytorch/test-infra
47+
test-infra-ref: main
48+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
49+
trigger-event: ${{ github.event_name }}
50+
package-name: pytorch_tokenizers
51+
smoke-test-script: .ci/scripts/wheel/test_wheel.py

0 commit comments

Comments
 (0)