[DOP-13277] Collect coverage results

.github/workflows/hdfs-tests.yml

@@ -76,10 +76,13 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m hdfs
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run pytest -vvv -s -m hdfs
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+      - name: Generate coverate reports
+        run: ./combine_coverage.sh
+
       - name: Shutdown
         if: always()
         run: |

.github/workflows/hive-tests.yml

@@ -76,10 +76,13 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m hive
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run pytest -vvv -s -m hive
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+      - name: Generate coverate reports
+        run: ./combine_coverage.sh
+
       - name: Shutdown
         if: always()
         run: |

.github/workflows/oracle-tests.yml

@@ -76,10 +76,13 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m oracle
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run pytest -vvv -s -m oracle
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+      - name: Generate coverate reports
+        run: ./combine_coverage.sh
+
       - name: Shutdown
         if: always()
         run: |

.github/workflows/s3-tests.yml

@@ -76,10 +76,13 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m s3
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run pytest -vvv -s -m s3
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+      - name: Generate coverate reports
+        run: ./combine_coverage.sh
+
       - name: Shutdown
         if: always()
         run: |

.github/workflows/unit-test.yml

@@ -57,7 +57,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          poetry install --no-root --with backend,test
+          poetry install --no-root --with backend,test,dev
 
       - name: Start docker compose
         run: |
@@ -69,7 +69,6 @@ jobs:
       - name: Run DB Tests.
         run: |
           source .env.local
-          mkdir -p reports/
           poetry run coverage run -m pytest -vvv -s ./syncmaster/tests/test_database
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
@@ -81,6 +80,9 @@ jobs:
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
+      - name: Generate coverate reports
+        run: ./combine_coverage.sh
+
       - name: Shutdown
         if: always()
         run: |

combine_coverage.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+root_path=$(dirname $(realpath $0))
+coverage combine --rcfile=tests/.coveragerc
+coverage xml --rcfile=tests/.coveragerc -o $root_path/reports/coverage.xml -i

docker-compose.test.yml

@@ -29,6 +29,7 @@ services:
       - test-s3
     volumes:
       - ./cached_jars:/root/.ivy2
+      - ./reports:/syncmaster/reports
     env_file: .env.docker
 
   backend:

docker/worker.dockerfile

@@ -26,14 +26,14 @@ COPY ./pyproject.toml ./poetry.lock* /syncmaster/
 
 RUN pip install --upgrade pip setuptools wheel packaging
 
-RUN poetry install --no-root --with worker,test
+RUN poetry install --no-root --with worker,test,dev
 
 COPY ./syncmaster/ /syncmaster/
 
 
 # https://docs.celeryq.dev/en/stable/userguide/workers.html#max-tasks-per-child-setting
 # Required to start each Celery task in separated process, avoiding issues with global Spark session object
-CMD ["celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1"]
+CMD ["coverage", "run", "celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1"]
 
 FROM prod as test
 

pyproject.toml

@@ -39,6 +39,7 @@ faker = "^22.6.0"
 
 [tool.poetry.group.dev.dependencies]
 coverage = "^7.4.3"
+gevent = "^24.2.1"
 pre-commit = "^3.3.3"
 mypy = "^1.4.1"
 black = "^23.3.0"
@@ -120,3 +121,33 @@ markers = [
     "hdfs: tests for hadoop hdfs",
     "s3: tests for S3",
 ]
+
+[tool.coverage.paths]
+source = ["syncmaster"]
+
+[tool.coverage.run]
+branch = true
+parallel = true
+relative_files = true
+concurrency = ["gevent"]
+omit = [
+  "tests/*",
+  "app/db/migrations/*",
+]
+data_file = "reports/.coverage"
+
+[tool.coverage.report]
+precision = 2
+exclude_lines = [
+  "pragma: no cover",
+  "def __repr__",
+  "if .*debug:",
+  "raise AssertionError",
+  "raise NotImplementedError",
+  "if __name__ == .__main__.:",
+  "if TYPE_CHECKING:",
+  "if log.isEnabledFor(logging.DEBUG):",
+  "if sys.version_info",
+  "@(abc\\.)?abstractmethod",
+  "\\.\\.\\.",
+]