[DOP-13277] Collect coverage results

.github/workflows/hdfs-tests.yml

@@ -1,16 +1,6 @@
 name: HDFS Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -76,10 +66,16 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m hdfs
+          docker compose -f ./docker-compose.test.yml exec -T worker python -m coverage run -m pytest -vvv -s -m hdfs
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: hdfs-tests
+          path: reports/*
+
       - name: Shutdown
         if: always()
         run: |

.github/workflows/hive-tests.yml

@@ -1,16 +1,6 @@
 name: Hive Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -76,10 +66,17 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m hive
+          docker compose -f ./docker-compose.test.yml exec -T worker python -m coverage run -m pytest -vvv -s -m hive
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: hive-tests
+          path: reports/*
+
       - name: Shutdown
         if: always()
         run: |

.github/workflows/oracle-tests.yml

@@ -1,16 +1,6 @@
 name: Oracle Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -76,10 +66,16 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m oracle
+          docker compose -f ./docker-compose.test.yml exec -T worker python -m coverage run -m pytest -vvv -s -m oracle
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: oracle-tests
+          path: reports/*
       - name: Shutdown
         if: always()
         run: |

.github/workflows/s3-tests.yml

@@ -1,16 +1,6 @@
 name: S3 Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -76,10 +66,16 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m s3
+          docker compose -f ./docker-compose.test.yml exec -T worker python -m coverage run -m pytest -vvv -s -m s3
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: s3-tests
+          path: reports/*
       - name: Shutdown
         if: always()
         run: |

.github/workflows/tests.yml

@@ -0,0 +1,73 @@
+name: Run All Tests
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+    branches-ignore:
+      - master
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEFAULT_PYTHON: '3.11'
+
+jobs:
+  oracle_tests:
+    name: Oracle tests
+    uses: ./.github/workflows/oracle-tests.yml
+
+  hive_tests:
+    name: Hive tests
+    uses: ./.github/workflows/hive-tests.yml
+
+  s3_tests:
+    name: S3 tests
+    uses: ./.github/workflows/s3-tests.yml
+
+  unit_tests:
+    name: Unit tests
+    uses: ./.github/workflows/unit-test.yml
+
+  all_done:
+    name: Tests done
+    runs-on: ubuntu-latest
+
+    env:
+      POETRY_VERSION: 1.6.1
+
+    needs: [oracle_tests, hive_tests, s3_tests, unit_tests]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-coverage
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip setuptools wheel
+
+      - name: Install dependencies
+        run: pip install -I coverage pytest
+
+      - name: Download all coverage reports
+        uses: actions/download-artifact@v4
+        with:
+          path: reports
+
+      - name: Move coverage data to the root folder
+        run: find reports -type f -exec mv '{}' reports \;
+
+      - name: Generate coverate reports
+        run: ./combine_coverage.sh

.github/workflows/unit-test.yml

@@ -1,17 +1,6 @@
 name: Unit Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
-
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -57,7 +46,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          poetry install --no-root --with backend,test
+          poetry install --no-root --with backend,test,dev
 
       - name: Start docker compose
         run: |
@@ -66,21 +55,19 @@ jobs:
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
-      - name: Run DB Tests.
-        run: |
-          source .env.local
-          mkdir -p reports/
-          poetry run coverage run -m pytest -vvv -s ./syncmaster/tests/test_database
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
-
       - name: Run Unit Tests.
         run: |
           source .env.local
-          poetry run coverage run -m pytest -vvv -s ./syncmaster/tests/test_unit
+          ./pytest_runner.sh -m backend
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit-tests
+          path: reports/*
+
       - name: Shutdown
         if: always()
         run: |

combine_coverage.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+root_path=$(dirname $(realpath $0))
+coverage combine --rcfile=syncmaster/tests/.coveragerc
+coverage xml --rcfile=syncmaster/tests/.coveragerc -o $root_path/reports/coverage.xml -i

docker-compose.test.yml

@@ -29,6 +29,7 @@ services:
       - test-s3
     volumes:
       - ./cached_jars:/root/.ivy2
+      - ./reports:/syncmaster/reports
     env_file: .env.docker
 
   backend:
@@ -52,7 +53,9 @@ services:
       - test-s3
     volumes:
       - ./cached_jars:/root/.ivy2
+      - ./reports:/syncmaster/reports
     env_file: .env.docker
+    command: [coverage, run, app/main.py]
 
   rabbitmq:
     image: rabbitmq

docker/worker.dockerfile

@@ -39,4 +39,4 @@ FROM prod as test
 
 ENV CREATE_SPARK_SESSION_FUNCTION="tests.spark.get_worker_spark_session.get_worker_spark_session"
 # Queue for tests
-CMD ["celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"]
+CMD ["coverage", "run", "-m", "celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"]