diff --git a/.github/actions/run-python-tests/action.yaml b/.github/actions/run-python-tests/action.yaml new file mode 100644 index 00000000000..9fc10b9a2e4 --- /dev/null +++ b/.github/actions/run-python-tests/action.yaml @@ -0,0 +1,73 @@ +name: "Run Python Tests" +description: "Setup and run Python tests" + +inputs: + python-version: + description: "Python version to use" + required: true + test-path: + description: "Path to the test file or directory" + required: true + test-name: + description: "Name of the test" + required: true + buildkite-analytics-token: + description: "Buildkite analytics token" + required: true + +runs: + using: "composite" + steps: + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + python-version: ${{ inputs.python-version }} + - name: Install dependencies + shell: bash + run: | + uv venv --seed ~/test-env + source ~/test-env/bin/activate + uv pip install --prerelease=allow "azure-cli>=2.65.0" + # Use -e to include examples and tests folder in the path for unit + # tests to access them. + uv pip install -e ".[all]" + uv pip install pytest pytest-xdist pytest-env>=0.6 pytest-asyncio memory-profiler==0.61.0 buildkite-test-collector + - name: Set branch name and commit message + id: set_env + shell: bash + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "branch=${{ github.head_ref }}" >> $GITHUB_OUTPUT + { + echo "message<> $GITHUB_OUTPUT + else + echo "branch=${{ github.ref_name }}" >> $GITHUB_OUTPUT + # For push events after merge, head_commit.message contains the merge commit message + # which includes the PR title. For merge_group events, use merge_group.head_commit.message + { + echo "message<> $GITHUB_OUTPUT + fi + - name: Run tests with pytest + shell: bash + env: + TEST_PATH: ${{ inputs.test-path }} + TEST_NAME: ${{ inputs.test-name }} + BUILDKITE_ANALYTICS_TOKEN: ${{ inputs.buildkite-analytics-token }} + run: | + source ~/test-env/bin/activate + if [[ "$TEST_NAME" == "No Parallel Tests" || "$TEST_NAME" == "Event Loop Lag Tests" ]]; then + SKYPILOT_DISABLE_USAGE_COLLECTION=1 SKYPILOT_SKIP_CLOUD_IDENTITY_CHECK=1 eval "pytest -n 0 --dist no $TEST_PATH" + else + SKYPILOT_DISABLE_USAGE_COLLECTION=1 SKYPILOT_SKIP_CLOUD_IDENTITY_CHECK=1 eval "pytest -n 4 --dist worksteal $TEST_PATH" + fi diff --git a/.github/workflows/pytest-optimizer.yml b/.github/workflows/pytest-optimizer.yml new file mode 100644 index 00000000000..3551d7d0361 --- /dev/null +++ b/.github/workflows/pytest-optimizer.yml @@ -0,0 +1,41 @@ +name: Python Tests +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - master + - 'releases/**' + pull_request: + branches: + - master + - 'releases/**' + merge_group: + +jobs: + python-test-optimizer: + strategy: + matrix: + python-version: ["3.8"] + test-path: + - "tests/test_optimizer_dryruns.py -k \"partial\"" + - "tests/test_optimizer_dryruns.py -k \"not partial\"" + - tests/test_optimizer_random_dag.py + include: + - test-path: "tests/test_optimizer_dryruns.py -k \"partial\"" + test-name: "Optimizer Dryruns Part 1" + - test-path: "tests/test_optimizer_dryruns.py -k \"not partial\"" + test-name: "Optimizer Dryruns Part 2" + - test-path: tests/test_optimizer_random_dag.py + test-name: "Optimizer Random DAG Tests" + runs-on: ubuntu-latest + name: "Python Tests - ${{ matrix.test-name }}" + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Run Python Tests + uses: ./.github/actions/run-python-tests + with: + python-version: ${{ matrix.python-version }} + test-path: ${{ matrix.test-path }} + test-name: ${{ matrix.test-name }} \ No newline at end of file diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2259a82376d..65086c09c1c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -21,11 +21,8 @@ jobs: # Group them based on running time to save CI time and resources - tests/unit_tests - tests/test_cli.py - - "tests/test_optimizer_dryruns.py -k \"partial\"" - - "tests/test_optimizer_dryruns.py -k \"not partial\"" - tests/test_jobs_and_serve.py tests/test_yaml_parser.py tests/test_global_user_state.py tests/test_config.py tests/test_jobs.py tests/test_list_accelerators.py tests/test_wheels.py tests/test_api.py tests/test_storage.py tests/test_api_compatibility.py - tests/test_no_parellel.py - - tests/test_optimizer_random_dag.py - tests/test_ssh_proxy_lag.py include: # We separate out the random DAG tests because its flaky due to catalog updates. @@ -34,16 +31,10 @@ jobs: test-name: "Unit Tests" - test-path: tests/test_cli.py test-name: "CLI Tests" - - test-path: "tests/test_optimizer_dryruns.py -k \"partial\"" - test-name: "Optimizer Dryruns Part 1" - - test-path: "tests/test_optimizer_dryruns.py -k \"not partial\"" - test-name: "Optimizer Dryruns Part 2" - test-path: tests/test_jobs_and_serve.py tests/test_yaml_parser.py tests/test_global_user_state.py tests/test_config.py tests/test_jobs.py tests/test_list_accelerators.py tests/test_wheels.py tests/test_api.py tests/test_storage.py tests/test_api_compatibility.py tests/test_infra_k8s_alias.py test-name: "Jobs, Serve, Wheels, API, Config, Optimizer & Storage Tests" - test-path: tests/test_no_parellel.py test-name: "No Parallel Tests" - - test-path: tests/test_optimizer_random_dag.py - test-name: "Optimizer Random DAG Tests" - test-path: tests/test_ssh_proxy_lag.py test-name: "Event Loop Lag Tests" runs-on: ubuntu-latest @@ -51,58 +42,13 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v4 + - name: Run Python Tests + uses: ./.github/actions/run-python-tests with: - version: "latest" python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - uv venv --seed ~/test-env - source ~/test-env/bin/activate - uv pip install --prerelease=allow "azure-cli>=2.65.0" - # Use -e to include examples and tests folder in the path for unit - # tests to access them. - uv pip install -e ".[all]" - uv pip install pytest pytest-xdist pytest-env>=0.6 pytest-asyncio memory-profiler==0.61.0 buildkite-test-collector - - name: Set branch name and commit message - id: set_env - run: | - if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "branch=${{ github.head_ref }}" >> $GITHUB_OUTPUT - { - echo "message<> $GITHUB_OUTPUT - else - echo "branch=${{ github.ref_name }}" >> $GITHUB_OUTPUT - # For push events after merge, head_commit.message contains the merge commit message - # which includes the PR title. For merge_group events, use merge_group.head_commit.message - { - echo "message<> $GITHUB_OUTPUT - fi - - name: Run tests with pytest - env: - TEST_PATH: ${{ matrix.test-path }} - TEST_NAME: ${{ matrix.test-name }} - BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} - run: | - source ~/test-env/bin/activate - export GITHUB_REF="${{ steps.set_env.outputs.branch }}" - export TEST_ANALYTICS_COMMIT_MESSAGE="${{ steps.set_env.outputs.message }}" - if [[ "$TEST_NAME" == "No Parallel Tests" || "$TEST_NAME" == "Event Loop Lag Tests" ]]; then - SKYPILOT_DISABLE_USAGE_COLLECTION=1 SKYPILOT_SKIP_CLOUD_IDENTITY_CHECK=1 eval "pytest -n 0 --dist no $TEST_PATH" - else - SKYPILOT_DISABLE_USAGE_COLLECTION=1 SKYPILOT_SKIP_CLOUD_IDENTITY_CHECK=1 eval "pytest -n 4 --dist worksteal $TEST_PATH" - fi + test-path: ${{ matrix.test-path }} + test-name: ${{ matrix.test-name }} + buildkite-analytics-token: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} limited-deps-test: # Test with limited dependencies to ensure cloud module imports don't break