[DOP-13487] Implement HDFS tests in github (#7)

* [DOP-13487] Implement HDFS tests in github

* [DOP-13487] Implement HDFS tests in github

* [DOP-13487] Implement HDFS tests in github

* [DOP-13487] Implement HDFS tests in github

.github/workflows/hdfs-tests.yml

@@ -0,0 +1,88 @@
+name: HDFS Tests
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+    branches-ignore:
+      - master
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEFAULT_PYTHON: '3.11'
+
+jobs:
+  tests:
+    name: Run HDFS tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+
+      - name: Cache jars
+        uses: actions/cache@v4
+        with:
+          path: ./cached_jars
+          key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-hdfs-jars
+          restore-keys: |
+            ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-hdfs-jars
+            ${{ runner.os }}-python
+            ${{ runner.os }}
+
+      - name: Build Worker Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          tags: syncmaster_worker:${{ github.sha }}
+          target: test
+          file: docker/worker.dockerfile
+          load: true
+          cache-to: type=gha,mode=max
+          cache-from: type=gha
+
+      - name: Docker compose up
+        run: |
+          docker compose -f docker-compose.test.yml down -v --remove-orphans
+          docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-hive test-postgres --wait --wait-timeout 200
+        env:
+          WORKER_IMAGE_TAG: ${{ github.sha }}
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Run HDFS Tests.
+        # Tests for the backend are run on the worker.
+        # Backend and worker on the same container.
+        run: |
+          docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m hdfs
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Shutdown
+        if: always()
+        run: |
+          docker compose -f docker-compose.test.yml down -v --remove-orphans
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster

.github/workflows/hive-tests.yml

@@ -58,6 +58,7 @@ jobs:
         with:
           context: .
           tags: syncmaster_worker:${{ github.sha }}
+          target: test
           file: docker/worker.dockerfile
           load: true
           cache-to: type=gha,mode=max

.github/workflows/oracle-tests.yml

@@ -58,6 +58,7 @@ jobs:
         with:
           context: .
           tags: syncmaster_worker:${{ github.sha }}
+          target: test
           file: docker/worker.dockerfile
           load: true
           cache-to: type=gha,mode=max

.github/workflows/s3-tests.yml

@@ -59,6 +59,7 @@ jobs:
           context: .
           tags: syncmaster_worker:${{ github.sha }}
           file: docker/worker.dockerfile
+          target: test
           load: true
           cache-to: type=gha,mode=max
           cache-from: type=gha

syncmaster/tests/test_integration/test_run_transfer/conftest.py

@@ -313,7 +313,7 @@ def hdfs_file_df_connection(spark, hdfs_server):
     from onetl.connection import SparkHDFS
 
     return SparkHDFS(
-        cluster="rnd-dwh",
+        cluster="test-hive",
         host=hdfs_server.host,
         ipc_port=hdfs_server.ipc_port,
         spark=spark,