Feat/add batch emails #10946
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR Tests - Syft | |
| on: | |
| workflow_call: | |
| pull_request: | |
| branches: | |
| - dev | |
| - main | |
| - "0.8" | |
| workflow_dispatch: | |
| inputs: | |
| none: | |
| description: "Run Tests Manually" | |
| required: false | |
| concurrency: | |
| group: syft-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.workflow, github.event.pull_request.number) || github.workflow_ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pr-tests-syft-unit: | |
| strategy: | |
| max-parallel: 99 | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| python-version: ["3.12"] | |
| include: | |
| - python-version: "3.11" | |
| os: "ubuntu-latest" | |
| - python-version: "3.10" | |
| os: "ubuntu-latest" | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| # - name: Permission to home directory | |
| # if: matrix.os == 'ubuntu-latest' | |
| # run: | | |
| # sudo chown -R $USER:$USER $HOME | |
| - name: "clean .git/config" | |
| if: matrix.os == 'windows-latest' | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| echo "deleting ${GITHUB_WORKSPACE}/.git/config" | |
| rm ${GITHUB_WORKSPACE}/.git/config | |
| - uses: actions/checkout@v4 | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.syft == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pip packages | |
| if: steps.changes.outputs.syft == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2 | |
| uv --version | |
| - name: Get uv cache dir | |
| id: pip-cache | |
| if: steps.changes.outputs.syft == 'true' | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: Load github cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.syft == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }}- | |
| # - name: Docker on MacOS | |
| # if: steps.changes.outputs.syft == 'true' && matrix.os == 'macos-latest' | |
| # uses: crazy-max/[email protected] | |
| # with: | |
| # set-host: true | |
| - name: Run unit tests | |
| if: steps.changes.outputs.syft == 'true' | |
| run: | | |
| tox -e syft.test.unit | |
| tox -e seaweedfs.test.unit | |
| pr-tests-syft-notebook-python: | |
| strategy: | |
| max-parallel: 99 | |
| matrix: | |
| # Disable on windows until its flakyness is reduced. | |
| # os: [ubuntu-latest, macos-latest, windows-latest] | |
| os: [ubuntu-latest, macos-latest] | |
| python-version: ["3.12"] | |
| deployment-type: ["python"] | |
| notebook-paths: ["tutorials"] | |
| bump-version: ["False"] | |
| include: | |
| - python-version: "3.11" | |
| os: "ubuntu-latest" | |
| deployment-type: "python" | |
| notebook-paths: "tutorials" | |
| - python-version: "3.10" | |
| os: "ubuntu-latest" | |
| deployment-type: "python" | |
| notebook-paths: "tutorials" | |
| - python-version: "3.12" | |
| os: "ubuntu-latest" | |
| deployment-type: "python" | |
| notebook-paths: "tutorials" | |
| bump-version: "True" | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| # - name: Permission to home directory | |
| # if: matrix.os == 'ubuntu-latest' | |
| # run: | | |
| # sudo chown -R $USER:$USER $HOME | |
| - name: "clean .git/config" | |
| if: matrix.os == 'windows-latest' | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| echo "deleting ${GITHUB_WORKSPACE}/.git/config" | |
| rm ${GITHUB_WORKSPACE}/.git/config | |
| - uses: actions/checkout@v4 | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pip packages | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2 | |
| uv --version | |
| - name: Get uv cache dir | |
| id: pip-cache | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true' | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: Load github cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }}- | |
| - name: Run notebook tests | |
| uses: nick-fields/retry@v3 | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true' | |
| env: | |
| ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}" | |
| TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}" | |
| BUMP_VERSION: "${{ matrix.bump-version }}" | |
| with: | |
| timeout_seconds: 2400 | |
| max_attempts: 3 | |
| command: tox -e syft.test.notebook | |
| pr-tests-syft-notebook-scenario: | |
| strategy: | |
| max-parallel: 99 | |
| matrix: | |
| # Disable on windows until its flakyness is reduced. | |
| # os: [ubuntu-latest, macos-latest, windows-latest] | |
| # os: [ubuntu-latest, macos-latest] | |
| os: [ubuntu-latest] # mac wont start SMTPD mail server? | |
| python-version: ["3.12"] | |
| deployment-type: ["python"] | |
| bump-version: ["False"] | |
| include: | |
| - python-version: "3.11" | |
| os: "ubuntu-latest" | |
| deployment-type: "python" | |
| - python-version: "3.10" | |
| os: "ubuntu-latest" | |
| deployment-type: "python" | |
| - python-version: "3.12" | |
| os: "ubuntu-latest" | |
| deployment-type: "python" | |
| bump-version: "True" | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| # - name: Permission to home directory | |
| # if: matrix.os == 'ubuntu-latest' | |
| # run: | | |
| # sudo chown -R $USER:$USER $HOME | |
| - name: "clean .git/config" | |
| if: matrix.os == 'windows-latest' | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| echo "deleting ${GITHUB_WORKSPACE}/.git/config" | |
| rm ${GITHUB_WORKSPACE}/.git/config | |
| - uses: actions/checkout@v4 | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pip packages | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2 | |
| uv --version | |
| - name: Get uv cache dir | |
| id: pip-cache | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: Load github cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }}- | |
| - name: Run notebook scenario tests | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| env: | |
| ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}" | |
| BUMP_VERSION: "${{ matrix.bump-version }}" | |
| TOX_PYTHON: python${{ matrix.python-version }} | |
| shell: bash | |
| run: | | |
| export PATH="/usr/share/miniconda/bin:$PATH" | |
| tox -e syft.test.notebook.scenario | |
| pr-tests-syft-notebook-scenario-sync: | |
| strategy: | |
| max-parallel: 99 | |
| matrix: | |
| # Disable on windows until its flakyness is reduced. | |
| # os: [ubuntu-latest, macos-latest, windows-latest] | |
| os: [ubuntu-latest, macos-latest] | |
| python-version: ["3.12"] | |
| deployment-type: ["python"] | |
| bump-version: ["False"] | |
| include: | |
| - python-version: "3.11" | |
| os: "ubuntu-latest" | |
| deployment-type: "python" | |
| - python-version: "3.10" | |
| os: "ubuntu-latest" | |
| deployment-type: "python" | |
| - python-version: "3.12" | |
| os: "ubuntu-latest" | |
| deployment-type: "python" | |
| bump-version: "True" | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| # - name: Permission to home directory | |
| # if: matrix.os == 'ubuntu-latest' | |
| # run: | | |
| # sudo chown -R $USER:$USER $HOME | |
| - name: "clean .git/config" | |
| if: matrix.os == 'windows-latest' | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| echo "deleting ${GITHUB_WORKSPACE}/.git/config" | |
| rm ${GITHUB_WORKSPACE}/.git/config | |
| - uses: actions/checkout@v4 | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pip packages | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2 | |
| uv --version | |
| - name: Get uv cache dir | |
| id: pip-cache | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: Load github cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }}- | |
| - name: Run notebook scenario with sync tests | |
| uses: nick-fields/retry@v3 | |
| if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks_scenario == 'true' | |
| env: | |
| ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}" | |
| BUMP_VERSION: "${{ matrix.bump-version }}" | |
| with: | |
| timeout_seconds: 2400 | |
| max_attempts: 3 | |
| command: tox -e syft.test.notebook.scenario.sync | |
| pr-tests-syft-notebook-single-container: | |
| strategy: | |
| max-parallel: 99 | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.10", "3.11", "3.12"] | |
| deployment-type: ["remote"] | |
| notebook-paths: ["api/0.8"] | |
| fail-fast: false | |
| runs-on: ${{matrix.os}} | |
| steps: | |
| # - name: Permission to home directory | |
| # if: matrix.os == 'ubuntu-latest' | |
| # run: | | |
| # sudo chown -R $USER:$USER $HOME | |
| - uses: actions/checkout@v4 | |
| # free 10GB of space | |
| - name: Remove unnecessary files | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| docker image prune --all --force | |
| docker builder prune --all --force | |
| docker system prune --all --force | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pip packages | |
| if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2 | |
| uv --version | |
| - name: Get uv cache dir | |
| id: pip-cache | |
| if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true' | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: Load github cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }}- | |
| - name: Docker Compose on Linux | |
| if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'ubuntu-latest' | |
| shell: bash | |
| run: | | |
| mkdir -p ~/.docker/cli-plugins | |
| DOCKER_COMPOSE_VERSION=v2.21.0 | |
| curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose | |
| chmod +x ~/.docker/cli-plugins/docker-compose | |
| docker compose version | |
| - name: Docker on MacOS | |
| if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest' | |
| uses: crazy-max/[email protected] | |
| - name: Docker Compose on MacOS | |
| if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest' | |
| shell: bash | |
| run: | | |
| brew install docker-compose | |
| mkdir -p ~/.docker/cli-plugins | |
| ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose || true | |
| docker compose version | |
| - name: Docker on Windows | |
| if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'windows-latest' | |
| shell: pwsh | |
| run: | | |
| [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 | |
| Start-BitsTransfer -Source "https://download.docker.com/win/static/stable/x86_64/docker-23.0.1.zip" | |
| Expand-Archive docker-23.0.1.zip -DestinationPath $Env:ProgramFiles | |
| &$Env:ProgramFiles\Docker\dockerd --register-service | |
| Start-Service docker | |
| docker version | |
| docker compose version | |
| - name: Run unit tests | |
| if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true' | |
| env: | |
| ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}" | |
| DEV_MODE: "True" # force orchestra --build | |
| TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}" | |
| run: | | |
| tox -e single_container.launch | |
| tox -e stack.test.notebook | |
| tox -e single_container.destroy | |
| pr-tests-syft-security: | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.12"] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| # - name: Permission to home directory | |
| # if: matrix.os == 'ubuntu-latest' | |
| # run: | | |
| # sudo chown -R $USER:$USER $HOME | |
| - uses: actions/checkout@v4 | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.syft == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pip packages | |
| if: steps.changes.outputs.syft == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install uv==0.4.1 tox==4.18.0 tox-uv==1.11.2 | |
| uv --version | |
| - name: Get uv cache dir | |
| if: steps.changes.outputs.syft == 'true' | |
| id: pip-cache | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: Load github cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.syft == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }}- | |
| - name: Scan for security issues | |
| if: steps.changes.outputs.syft == 'true' | |
| run: | | |
| tox -e syft.test.security |