[DOP-13277] Collect coverage results (#8)

* [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results Co-authored-by: Maxim Martynov <[email protected]> * [DOP-13277] Collect coverage results * [DOP-13277] Collect coverage results --------- Co-authored-by: Maxim Martynov <[email protected]>
MobileTeleSystems · Mar 11, 2024 · 0062300 · 0062300
1 parent bffee9f
commit 0062300
Show file tree

Hide file tree

Showing 13 changed files with 410 additions and 134 deletions.
diff --git a/.github/workflows/hdfs-tests.yml b/.github/workflows/hdfs-tests.yml
@@ -1,16 +1,6 @@
 name: HDFS Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -76,7 +66,7 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m hdfs
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m hdfs
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
@@ -86,3 +76,9 @@ jobs:
           docker compose -f docker-compose.test.yml down -v --remove-orphans
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: hdfs-tests
+          path: reports/*
diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml
@@ -1,16 +1,6 @@
 name: Hive Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -76,7 +66,7 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m hive
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m hive
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
@@ -86,3 +76,9 @@ jobs:
           docker compose -f docker-compose.test.yml down -v --remove-orphans
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: hive-tests
+          path: reports/*
diff --git a/.github/workflows/oracle-tests.yml b/.github/workflows/oracle-tests.yml
@@ -1,16 +1,6 @@
 name: Oracle Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -76,7 +66,7 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m oracle
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m oracle
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
@@ -86,3 +76,9 @@ jobs:
           docker compose -f docker-compose.test.yml down -v --remove-orphans
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: oracle-tests
+          path: reports/*
diff --git a/.github/workflows/s3-tests.yml b/.github/workflows/s3-tests.yml
@@ -1,16 +1,6 @@
 name: S3 Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -76,7 +66,7 @@ jobs:
         # Tests for the backend are run on the worker.
         # Backend and worker on the same container.
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m s3
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m s3
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
@@ -86,3 +76,9 @@ jobs:
           docker compose -f docker-compose.test.yml down -v --remove-orphans
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: s3-tests
+          path: reports/*
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -0,0 +1,84 @@
+name: Run All Tests
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+    branches-ignore:
+      - master
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEFAULT_PYTHON: '3.11'
+
+jobs:
+  oracle_tests:
+    name: Oracle tests
+    uses: ./.github/workflows/oracle-tests.yml
+
+  hive_tests:
+    name: Hive tests
+    uses: ./.github/workflows/hive-tests.yml
+
+  s3_tests:
+    name: S3 tests
+    uses: ./.github/workflows/s3-tests.yml
+
+  unit_tests:
+    name: Unit tests
+    uses: ./.github/workflows/unit-test.yml
+
+  all_done:
+    name: Tests done
+    runs-on: ubuntu-latest
+
+    env:
+      POETRY_VERSION: 1.6.1
+
+    needs: [oracle_tests, hive_tests, s3_tests, unit_tests]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-coverage
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip setuptools wheel
+
+      - name: Install dependencies
+        run: pip install -I coverage pytest
+
+      - name: Download all coverage reports
+        uses: actions/download-artifact@v4
+        with:
+          path: reports
+
+      - name: Move coverage data to the root folder
+        run: find reports -type f -exec mv '{}' reports \;
+
+      - name: Generate coverage reports
+        run: ./combine_coverage.sh
+
+      - name: Check coverage
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./reports/coverage.xml
+          fail_ci_if_error: true
+          plugin: noop
+
+      - name: All done
+        run: echo 1
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -1,17 +1,6 @@
 name: Unit Tests
 on:
-  push:
-    branches:
-      - develop
-  pull_request:
-    branches-ignore:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
-
+  workflow_call:
 
 env:
   DEFAULT_PYTHON: '3.11'
@@ -66,18 +55,10 @@ jobs:
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
-      - name: Run DB Tests.
-        run: |
-          source .env.local
-          mkdir -p reports/
-          poetry run coverage run -m pytest -vvv -s ./syncmaster/tests/test_database
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
-
       - name: Run Unit Tests.
         run: |
           source .env.local
-          poetry run coverage run -m pytest -vvv -s ./syncmaster/tests/test_unit
+          ./pytest_runner.sh -m backend
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
@@ -87,3 +68,9 @@ jobs:
           docker compose -f docker-compose.test.yml down -v --remove-orphans
         env:
           COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit-tests
+          path: reports/*
diff --git a/combine_coverage.sh b/combine_coverage.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+root_path=$(dirname $(realpath $0))
+coverage combine
+coverage xml -o $root_path/reports/coverage.xml -i
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -29,11 +29,11 @@ services:
       - test-s3
     volumes:
       - ./cached_jars:/root/.ivy2
+      - ./reports:/syncmaster/reports
     env_file: .env.docker
 
   backend:
     image: syncmaster_back
-    container_name: backend
     restart: unless-stopped
     build:
       dockerfile: docker/backend.dockerfile
@@ -52,7 +52,9 @@ services:
       - test-s3
     volumes:
       - ./cached_jars:/root/.ivy2
+      - ./reports:/syncmaster/reports
     env_file: .env.docker
+    command: [coverage, run, app/main.py]
 
   rabbitmq:
     image: rabbitmq

diff --git a/docker/worker.dockerfile b/docker/worker.dockerfile
@@ -26,7 +26,7 @@ COPY ./pyproject.toml ./poetry.lock* /syncmaster/
 
 RUN pip install --upgrade pip setuptools wheel packaging
 
-RUN poetry install --no-root --with worker,test
+RUN poetry install --no-root --with worker
 
 COPY ./syncmaster/ /syncmaster/
 
@@ -37,6 +37,8 @@ CMD ["celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--
 
 FROM prod as test
 
+RUN poetry install --no-root --with worker,test
+
 ENV CREATE_SPARK_SESSION_FUNCTION="tests.spark.get_worker_spark_session.get_worker_spark_session"
 # Queue for tests
-CMD ["celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"]
+CMD ["coverage", "run", "-m", "celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"]