main.yml

name: CI

# ensures that main has our basic test and code quality
on:
  push:
    branches: [main]

jobs:
  build-pex:
    runs-on: ubuntu-22.04

    strategy:
      matrix:
        python-version: ["3.9"]

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install ansible
          pip install virtualenv
          pip install wheel
          pip install pex

      - name: Repack confluent-kafka wheel
        run: |
          rm -rf tmp_pip_cache &&
          mkdir tmp_pip_cache &&
          cd tmp_pip_cache &&
          python -m pip download $(cat ../requirements.txt | grep confluent-kafka) &&
          unzip * &&
          rm *.whl &&
          python -m wheel pack .

      - name: Build Pex File
        run: |
          pex . -r requirements.txt -o ./logprep.pex -c logprep --pex-root=tmp_pip_cache

      - name: Upload PEX
        uses: actions/upload-artifact@v2
        with:
          name: Logprep
          path: logprep.pex

  test:
    runs-on: ubuntu-22.04

    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11"]

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"

      - name: Install dependencies
        run: |
          pip install --upgrade pip wheel
          pip install -r requirements_dev.txt

      - name: Perform unit tests
        env:
          PYTEST_ADDOPTS: "--color=yes"
        run: |
          pytest  -vv tests/unit

      - name: Perform acceptance tests
        env:
          PYTEST_ADDOPTS: "--color=yes"
        run: |
          pytest  -vv tests/acceptance

  code-quality:
    runs-on: ubuntu-22.04

    strategy:
      matrix:
        python-version: ["3.9"]

    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"

      - name: Get changed python files
        id: changed-files
        uses: tj-actions/changed-files@v18.7
        with:
          files: |
            **/*.py

      - name: Install dependencies
        run: |
          sudo apt-get update && sudo apt-get -y install libhyperscan-dev librdkafka-dev
          pip install --upgrade pip wheel
          pip install -r requirements_dev.txt

      - name: check black formating
        run: |
          black --check --diff --config ./pyproject.toml .

      - name: lint changed and added files
        if: steps.changed-files.outputs.all_changed_files
        run: |
          pylint --rcfile=.pylintrc --fail-under 9.5 ${{ steps.changed-files.outputs.all_changed_files }}

      - name: Run tests and collect coverage
        run: pytest tests/unit --cov=logprep --cov-report=xml

      - name: Upload coverage reports to Codecov with GitHub Action
        uses: codecov/codecov-action@v2

      - name: Check semgrep rules
        if: steps.changed-files.outputs.all_changed_files
        run: semgrep -c .semgrep_rules -c r/python --error --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }}