Use model cache for docker compose test (opea-project#1582)

Signed-off-by: ZePan110 <[email protected]>

Author: ZePan110

.github/workflows/_example-workflow.yml

@@ -43,7 +43,11 @@ on:
       inject_commit:
         default: false
         required: false
-        type: string
+        type: boolean
+      use_model_cache:
+        default: false
+        required: false
+        type: boolean
 
 jobs:
 ####################################################################################################
@@ -110,6 +114,7 @@ jobs:
       tag: ${{ inputs.tag }}
       example: ${{ inputs.example }}
       hardware: ${{ inputs.node }}
+      use_model_cache: ${{ inputs.use_model_cache }}
     secrets: inherit
 
 
@@ -131,7 +136,7 @@ jobs:
 ####################################################################################################
   test-gmc-pipeline:
     needs: [build-images]
-    if: ${{ fromJSON(inputs.test_gmc) }}
+    if: false # ${{ fromJSON(inputs.test_gmc) }}
     uses: ./.github/workflows/_gmc-e2e.yml
     with:
       example: ${{ inputs.example }}

.github/workflows/_run-docker-compose.yml

@@ -28,6 +28,10 @@ on:
         required: false
         type: string
         default: ""
+      use_model_cache:
+        required: false
+        type: boolean
+        default: false
 jobs:
   get-test-case:
     runs-on: ubuntu-latest
@@ -144,9 +148,11 @@ jobs:
           example: ${{ inputs.example }}
           hardware: ${{ inputs.hardware }}
           test_case: ${{ matrix.test_case }}
+          use_model_cache: ${{ inputs.use_model_cache }}
         run: |
           cd ${{ github.workspace }}/$example/tests
           if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi
+          if [[ "$use_model_cache" == "true" ]]; then export model_cache="/data2/hf_model"; fi
           if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi
 
       - name: Clean up container after test

.github/workflows/manual-example-workflow.yml

@@ -20,11 +20,11 @@ on:
         description: "Tag to apply to images"
         required: true
         type: string
-      deploy_gmc:
-        default: false
-        description: 'Whether to deploy gmc'
-        required: true
-        type: boolean
+      # deploy_gmc:
+      #   default: false
+      #   description: 'Whether to deploy gmc'
+      #   required: true
+      #   type: boolean
       build:
         default: true
         description: 'Build test required images for Examples'
@@ -40,11 +40,11 @@ on:
         description: 'Test examples with helm charts'
         required: false
         type: boolean
-      test_gmc:
-        default: false
-        description: 'Test examples with gmc'
-        required: false
-        type: boolean
+      # test_gmc:
+      #   default: false
+      #   description: 'Test examples with gmc'
+      #   required: false
+      #   type: boolean
       opea_branch:
         default: "main"
         description: 'OPEA branch for image build'
@@ -54,7 +54,12 @@ on:
         default: false
         description: "inject commit to docker images true or false"
         required: false
-        type: string
+        type: boolean
+      use_model_cache:
+        default: false
+        description: "use model cache true or false"
+        required: false
+        type: boolean
 
 permissions: read-all
 jobs:
@@ -76,7 +81,8 @@ jobs:
 
   build-deploy-gmc:
     needs: [get-test-matrix]
-    if: ${{ fromJSON(inputs.deploy_gmc) }}
+    if: false
+    #${{ fromJSON(inputs.deploy_gmc) }}
     strategy:
       matrix:
         node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }}
@@ -89,7 +95,7 @@ jobs:
     secrets: inherit
 
   run-examples:
-    needs: [get-test-matrix, build-deploy-gmc]
+    needs: [get-test-matrix]   #[get-test-matrix, build-deploy-gmc]
     if: always()
     strategy:
       matrix:
@@ -104,7 +110,8 @@ jobs:
       build: ${{ fromJSON(inputs.build) }}
       test_compose: ${{ fromJSON(inputs.test_compose) }}
       test_helmchart: ${{ fromJSON(inputs.test_helmchart) }}
-      test_gmc: ${{ fromJSON(inputs.test_gmc) }}
+      # test_gmc: ${{ fromJSON(inputs.test_gmc) }}
       opea_branch: ${{ inputs.opea_branch }}
       inject_commit: ${{ inputs.inject_commit }}
+      use_model_cache: ${{ inputs.use_model_cache }}
     secrets: inherit

.github/workflows/pr-docker-compose-e2e.yml

@@ -42,5 +42,6 @@ jobs:
       tag: "ci"
       example: ${{ matrix.example }}
       hardware: ${{ matrix.hardware }}
+      use_model_cache: true
       diff_excluded_files: '\.github|\.md|\.txt|kubernetes|gmc|assets|benchmark'
     secrets: inherit

AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml

@@ -30,7 +30,7 @@ services:
     ports:
       - "3006:80"
     volumes:
-      - "./data:/data"
+      - "${MODEL_CACHE}:/data"
     shm_size: 1g
     environment:
       no_proxy: ${no_proxy}

AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml

@@ -31,7 +31,7 @@ services:
     ports:
       - "3006:80"
     volumes:
-      - "./data:/data"
+      - "${MODEL_CACHE}:/data"
     shm_size: 1g
     environment:
       no_proxy: ${no_proxy}

AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -40,7 +40,7 @@ services:
     ports:
       - "3006:80"
     volumes:
-      - "./data:/data"
+      - "${MODEL_CACHE}:/data"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}

AudioQnA/tests/test_compose_on_gaudi.sh

@@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
 echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
 export REGISTRY=${IMAGE_REPO}
 export TAG=${IMAGE_TAG}
+export MODEL_CACHE=${model_cache:-"./data"}
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"

AudioQnA/tests/test_compose_on_xeon.sh

@@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
 echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
 export REGISTRY=${IMAGE_REPO}
 export TAG=${IMAGE_TAG}
+export MODEL_CACHE=${model_cache:-"./data"}
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"

DBQnA/docker_compose/intel/cpu/xeon/compose.yaml

@@ -8,7 +8,7 @@ services:
     ports:
       - "8008:80"
     volumes:
-      - "./data:/data"
+      - "${MODEL_CACHE}:/data"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}