Refactor : Backend API to integrate zenml integrations

.github/workflows/docker_build.yml

@@ -4,24 +4,23 @@ on:
     branches:
       - master
       - develop
-
     paths:
       - "backend/**"
       - ".github/workflows/docker_build.yml"
   pull_request:
     branches:
       - master
       - develop
-
     paths:
       - "backend/**"
       - ".github/workflows/docker_build.yml"
+
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-and-push-api-image:
+  build-api-image:
     runs-on: ubuntu-24.04
     permissions:
       contents: read
@@ -51,128 +50,22 @@ jobs:
             type=ref,event=tag
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=test
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build and push API Docker image
+      - name: Build API Docker image
         uses: docker/build-push-action@v6
         with:
           context: backend/
-          file: backend/Dockerfile.API
+          file: backend/Dockerfile
           platforms: linux/amd64
           provenance: false
           sbom: false
-          push: false
+          push: true
           tags: ${{ steps.meta_api.outputs.tags }}
           labels: ${{ steps.meta_api.outputs.labels }}
           cache-from: type=gha,scope=api,timeout=20m
           cache-to: type=gha,mode=max,scope=api,timeout=20m,ignore-error=true
           github-token: ${{ secrets.GITHUB_TOKEN }}
-
-  build-and-push-worker-image:
-    needs: build-and-push-api-image
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        build_type: [cpu, gpu]
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Remove unnecessary files
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-
-      - uses: actions/checkout@v5
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for Worker Docker
-        id: meta_worker
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/fair-worker
-          tags: |
-            type=ref,event=branch,suffix=-${{ matrix.build_type }}
-            type=ref,event=tag,suffix=-${{ matrix.build_type }}
-            type=semver,pattern={{version}},suffix=-${{ matrix.build_type }}
-            type=semver,pattern={{major}}.{{minor}},suffix=-${{ matrix.build_type }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push Worker Docker image (${{ matrix.build_type }})
-        uses: docker/build-push-action@v6
-        with:
-          context: backend/
-          file: backend/Dockerfile.workers
-          platforms: linux/amd64
-          provenance: false
-          sbom: false
-          push: false
-          build-args: |
-            BUILD_TYPE=${{ matrix.build_type }}
-          tags: ${{ steps.meta_worker.outputs.tags }}
-          labels: ${{ steps.meta_worker.outputs.labels }}
-          cache-from: type=gha,scope=docker-worker-${{ matrix.build_type }},timeout=20m
-          cache-to: type=gha,mode=min,scope=docker-worker-${{ matrix.build_type }},timeout=20m,ignore-error=true
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-  build-and-push-offline-predictor-image:
-    runs-on: ubuntu-24.04
-    needs: build-and-push-api-image
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Remove unnecessary files
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-
-      - uses: actions/checkout@v5
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for Offline Predictor Docker
-        id: meta_offline_predictor
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/fair-offline-predictor
-          tags: |
-            type=ref,event=branch
-            type=ref,event=tag
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push Offline Predictor Docker image
-        uses: docker/build-push-action@v6
-        with:
-          context: backend/
-          file: backend/Dockerfile.API
-          platforms: linux/amd64
-          provenance: false
-          sbom: false
-          push: false
-          build-args: |
-            BUILD_TARGET=predictor
-          tags: ${{ steps.meta_offline_predictor.outputs.tags }}
-          labels: ${{ steps.meta_offline_predictor.outputs.labels }}
-          cache-from: type=gha,scope=offline-predictor,timeout=20m
-          cache-to: type=gha,mode=max,scope=offline-predictor,timeout=20m,ignore-error=true
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docker_publish_image.yml

@@ -66,7 +66,7 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: backend/
-          file: backend/Dockerfile.API
+          file: backend/Dockerfile
           platforms: linux/amd64
           provenance: false
           sbom: false
@@ -91,142 +91,3 @@ jobs:
               (.image.os == "linux" and .image.architecture == "amd64")
             end
           ' >/dev/null
-
-  build-and-push-worker-image:
-    needs: build-and-push-api-image
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        build_type: [cpu, gpu]
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Remove unnecessary files
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-
-      - uses: actions/checkout@v5
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for Worker Docker
-        id: meta_worker
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/fair-worker
-          tags: |
-            type=ref,event=branch,suffix=-${{ matrix.build_type }}
-            type=ref,event=tag,suffix=-${{ matrix.build_type }}
-            type=semver,pattern={{version}},suffix=-${{ matrix.build_type }}
-            type=semver,pattern={{major}}.{{minor}},suffix=-${{ matrix.build_type }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push Worker Docker image (${{ matrix.build_type }})
-        id: build_worker
-        uses: docker/build-push-action@v6
-        with:
-          context: backend/
-          file: backend/Dockerfile.workers
-          platforms: linux/amd64
-          provenance: false
-          sbom: false
-          push: true
-          build-args: |
-            BUILD_TYPE=${{ matrix.build_type }}
-          tags: ${{ steps.meta_worker.outputs.tags }}
-          labels: ${{ steps.meta_worker.outputs.labels }}
-          cache-from: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.use_cache == 'false') && '' || format('type=gha,scope=docker-worker-{0},timeout=20m', matrix.build_type) }}
-          cache-to: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.use_cache == 'false') && '' || format('type=gha,mode=min,scope=docker-worker-{0},timeout=20m,ignore-error=true', matrix.build_type) }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Validate Worker manifest
-        env:
-          IMAGE_REF: ${{ env.REGISTRY }}/${{ github.repository_owner }}/fair-worker@${{ steps.build_worker.outputs.digest }}
-        run: |
-          json=$(docker buildx imagetools inspect --format '{{json .}}' "$IMAGE_REF")
-          echo "$json" | jq -e '
-            if (.manifest.manifests? | type) == "array" then
-              (any(.manifest.manifests[]?; .platform.os == "linux" and .platform.architecture == "amd64"))
-              and
-              (all(.manifest.manifests[]?; (.platform.os != "unknown" and .platform.architecture != "unknown")))
-            else
-              (.image.os == "linux" and .image.architecture == "amd64")
-            end
-          ' >/dev/null
-
-  build-and-push-offline-predictor-image:
-    runs-on: ubuntu-24.04
-    needs: build-and-push-api-image
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Remove unnecessary files
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-
-      - uses: actions/checkout@v5
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for Offline Predictor Docker
-        id: meta_offline_predictor
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/fair-offline-predictor
-          tags: |
-            type=ref,event=branch
-            type=ref,event=tag
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push Offline Predictor Docker image
-        id: build_offline_predictor
-        uses: docker/build-push-action@v6
-        with:
-          context: backend/
-          file: backend/Dockerfile.API
-          platforms: linux/amd64
-          provenance: false
-          sbom: false
-          push: true
-          build-args: |
-            BUILD_TARGET=predictor
-          tags: ${{ steps.meta_offline_predictor.outputs.tags }}
-          labels: ${{ steps.meta_offline_predictor.outputs.labels }}
-          cache-from: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.use_cache == 'false') && '' || 'type=gha,scope=offline-predictor,timeout=20m' }}
-          cache-to: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.use_cache == 'false') && '' || 'type=gha,mode=max,scope=offline-predictor,timeout=20m,ignore-error=true' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Validate Offline Predictor manifest
-        env:
-          IMAGE_REF: ${{ env.REGISTRY }}/${{ github.repository_owner }}/fair-offline-predictor@${{ steps.build_offline_predictor.outputs.digest }}
-        run: |
-          json=$(docker buildx imagetools inspect --format '{{json .}}' "$IMAGE_REF")
-          echo "$json" | jq -e '
-            if (.manifest.manifests? | type) == "array" then
-              (any(.manifest.manifests[]?; .platform.os == "linux" and .platform.architecture == "amd64"))
-              and
-              (all(.manifest.manifests[]?; (.platform.os != "unknown" and .platform.architecture != "unknown")))
-            else
-              (.image.os == "linux" and .image.architecture == "amd64")
-            end
-          ' >/dev/null