Skip to content

Commit ea17b38

Browse files
chensuyuechensuyue
authored
[CICD enhance] AudioQnA run CI with latest base image, group logs in GHA outputs. (opea-project#1854)
Signed-off-by: chensuyue <[email protected]>
1 parent ea9d444 commit ea17b38

13 files changed

+135
-256
lines changed

‎AgentQnA/tests/test_compose_on_gaudi.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,22 +88,21 @@ echo "::endgroup::"
8888

8989
cd $WORKPATH/tests
9090

91-
echo "::group::=================== #1 Building docker images===================="
91+
echo "::group::=================== Building docker images===================="
9292
bash step1_build_images.sh gaudi_vllm > docker_image_build.log
9393
echo "::endgroup::"
9494

95-
echo "::endgroup::=================== #4 Start agent, API server, retrieval, and ingest data===================="
95+
echo "::group::=================== Start agent, API server, retrieval, and ingest data===================="
9696
bash step4_launch_and_validate_agent_gaudi.sh
9797
echo "::endgroup::"
9898

99-
echo "::endgroup::=================== #5 Stop agent and API server===================="
99+
echo "::group::=================== Stop agent and API server===================="
100100
stop_llm
101101
stop_crag
102102
stop_agent_containers
103103
stop_retrieval_tool
104104
stop_telemetry_containers
105-
echo "::endgroup::"
106-
107105
echo y | docker system prune
106+
echo "::endgroup::"
108107

109108
echo "ALL DONE!!"

‎AudioQnA/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
56
FROM opea/comps-base:$BASE_TAG
7+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
68

79
COPY ./audioqna.py $HOME/audioqna.py
810

‎AudioQnA/Dockerfile.multilang‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
56
FROM opea/comps-base:$BASE_TAG
7+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
68

79
COPY ./audioqna_multilang.py $HOME/audioqna_multilang.py
810

‎AudioQnA/docker_image_build/build.yaml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ services:
55
audioqna:
66
build:
77
args:
8+
IMAGE_REPO: ${REGISTRY}
9+
BASE_TAG: ${TAG}
810
http_proxy: ${http_proxy}
911
https_proxy: ${https_proxy}
1012
no_proxy: ${no_proxy}
@@ -85,10 +87,7 @@ services:
8587
image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest}
8688
vllm-rocm:
8789
build:
88-
args:
89-
http_proxy: ${http_proxy}
90-
https_proxy: ${https_proxy}
91-
no_proxy: ${no_proxy}
9290
context: GenAIComps
9391
dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu
92+
extends: audioqna
9493
image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}

‎AudioQnA/tests/test_compose_multilang_on_xeon.sh‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
3327

3428
git clone https://github.com/vllm-project/vllm.git
3529
cd ./vllm/
@@ -103,14 +97,26 @@ function stop_docker() {
10397

10498
function main() {
10599

100+
echo "::group::stop_docker"
106101
stop_docker
102+
echo "::endgroup::"
103+
104+
echo "::group::build_docker_images"
107105
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
106+
echo "::endgroup::"
107+
108+
echo "::group::start_services"
108109
start_services
110+
echo "::endgroup::"
109111

112+
echo "::group::validate_megaservice"
110113
validate_megaservice
114+
echo "::endgroup::"
111115

116+
echo "::group::stop_docker"
112117
stop_docker
113-
echo y | docker system prune
118+
docker system prune -f
119+
echo "::endgroup::"
114120

115121
}
116122

‎AudioQnA/tests/test_compose_on_gaudi.sh‎

Lines changed: 17 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
3327

3428
git clone https://github.com/HabanaAI/vllm-fork.git
3529
cd vllm-fork/
@@ -105,50 +99,35 @@ function validate_megaservice() {
10599
echo "Result wrong."
106100
exit 1
107101
fi
108-
109102
}
110103

111-
#function validate_frontend() {
112-
# cd $WORKPATH/ui/svelte
113-
# local conda_env_name="OPEA_e2e"
114-
# export PATH=${HOME}/miniforge3/bin/:$PATH
115-
## conda remove -n ${conda_env_name} --all -y
116-
## conda create -n ${conda_env_name} python=3.12 -y
117-
# source activate ${conda_env_name}
118-
#
119-
# sed -i "s/localhost/$ip_address/g" playwright.config.ts
120-
#
121-
## conda install -c conda-forge nodejs=22.6.0 -y
122-
# npm install && npm ci && npx playwright install --with-deps
123-
# node -v && npm -v && pip list
124-
#
125-
# exit_status=0
126-
# npx playwright test || exit_status=$?
127-
#
128-
# if [ $exit_status -ne 0 ]; then
129-
# echo "[TEST INFO]: ---------frontend test failed---------"
130-
# exit $exit_status
131-
# else
132-
# echo "[TEST INFO]: ---------frontend test passed---------"
133-
# fi
134-
#}
135-
136104
function stop_docker() {
137105
cd $WORKPATH/docker_compose/intel/hpu/gaudi
138106
docker compose -f compose.yaml stop && docker compose rm -f
139107
}
140108

141109
function main() {
142110

111+
echo "::group::stop_docker"
143112
stop_docker
113+
echo "::endgroup::"
114+
115+
echo "::group::build_docker_images"
144116
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
117+
echo "::endgroup::"
118+
119+
echo "::group::start_services"
145120
start_services
121+
echo "::endgroup::"
146122

123+
echo "::group::validate_megaservice"
147124
validate_megaservice
148-
# validate_frontend
125+
echo "::endgroup::"
149126

127+
echo "::group::stop_docker"
150128
stop_docker
151-
echo y | docker system prune
129+
docker system prune -f
130+
echo "::endgroup::"
152131

153132
}
154133

‎AudioQnA/tests/test_compose_on_rocm.sh‎

Lines changed: 18 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -18,25 +18,18 @@ export PATH="~/miniconda3/bin:$PATH"
1818

1919
function build_docker_images() {
2020
opea_branch=${opea_branch:-"main"}
21-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
22-
if [[ "${opea_branch}" != "main" ]]; then
23-
cd $WORKPATH
24-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
25-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
26-
find . -type f -name "Dockerfile*" | while read -r file; do
27-
echo "Processing file: $file"
28-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
29-
done
30-
fi
3121

3222
cd $WORKPATH/docker_image_build
3323
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
24+
pushd GenAIComps
25+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
26+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
27+
popd && sleep 1s
3428

3529
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3630
service_list="audioqna audioqna-ui whisper speecht5"
3731
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
38-
echo "docker pull ghcr.io/huggingface/text-generation-inference:2.4.1-rocm"
39-
docker pull ghcr.io/huggingface/text-generation-inference:2.4.1-rocm
32+
4033
docker images && sleep 1s
4134
}
4235

@@ -56,8 +49,6 @@ function start_services() {
5649

5750
export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
5851

59-
# sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
60-
6152
# Start Docker Containers
6253
docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
6354
n=0
@@ -88,49 +79,33 @@ function validate_megaservice() {
8879

8980
}
9081

91-
#function validate_frontend() {
92-
# Frontend tests are currently disabled
93-
# cd $WORKPATH/ui/svelte
94-
# local conda_env_name="OPEA_e2e"
95-
# export PATH=${HOME}/miniforge3/bin/:$PATH
96-
## conda remove -n ${conda_env_name} --all -y
97-
## conda create -n ${conda_env_name} python=3.12 -y
98-
# source activate ${conda_env_name}
99-
#
100-
# sed -i "s/localhost/$ip_address/g" playwright.config.ts
101-
#
102-
## conda install -c conda-forge nodejs -y
103-
# npm install && npm ci && npx playwright install --with-deps
104-
# node -v && npm -v && pip list
105-
#
106-
# exit_status=0
107-
# npx playwright test || exit_status=$?
108-
#
109-
# if [ $exit_status -ne 0 ]; then
110-
# echo "[TEST INFO]: ---------frontend test failed---------"
111-
# exit $exit_status
112-
# else
113-
# echo "[TEST INFO]: ---------frontend test passed---------"
114-
# fi
115-
#}
116-
11782
function stop_docker() {
11883
cd $WORKPATH/docker_compose/amd/gpu/rocm/
11984
docker compose stop && docker compose rm -f
12085
}
12186

12287
function main() {
12388

89+
echo "::group::stop_docker"
12490
stop_docker
91+
echo "::endgroup::"
92+
93+
echo "::group::build_docker_images"
12594
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
95+
echo "::endgroup::"
96+
97+
echo "::group::start_services"
12698
start_services
99+
echo "::endgroup::"
127100

101+
echo "::group::validate_megaservice"
128102
validate_megaservice
129-
# Frontend tests are currently disabled
130-
# validate_frontend
103+
echo "::endgroup::"
131104

105+
echo "::group::stop_docker"
132106
stop_docker
133-
echo y | docker system prune
107+
docker system prune -f
108+
echo "::endgroup::"
134109

135110
}
136111

‎AudioQnA/tests/test_compose_on_xeon.sh‎

Lines changed: 17 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
3327

3428
git clone https://github.com/vllm-project/vllm.git
3529
cd ./vllm/
@@ -95,47 +89,33 @@ function validate_megaservice() {
9589

9690
}
9791

98-
#function validate_frontend() {
99-
# cd $WORKPATH/ui/svelte
100-
# local conda_env_name="OPEA_e2e"
101-
# export PATH=${HOME}/miniforge3/bin/:$PATH
102-
## conda remove -n ${conda_env_name} --all -y
103-
## conda create -n ${conda_env_name} python=3.12 -y
104-
# source activate ${conda_env_name}
105-
#
106-
# sed -i "s/localhost/$ip_address/g" playwright.config.ts
107-
#
108-
## conda install -c conda-forge nodejs=22.6.0 -y
109-
# npm install && npm ci && npx playwright install --with-deps
110-
# node -v && npm -v && pip list
111-
#
112-
# exit_status=0
113-
# npx playwright test || exit_status=$?
114-
#
115-
# if [ $exit_status -ne 0 ]; then
116-
# echo "[TEST INFO]: ---------frontend test failed---------"
117-
# exit $exit_status
118-
# else
119-
# echo "[TEST INFO]: ---------frontend test passed---------"
120-
# fi
121-
#}
122-
12392
function stop_docker() {
12493
cd $WORKPATH/docker_compose/intel/cpu/xeon/
12594
docker compose -f compose.yaml stop && docker compose rm -f
12695
}
12796

12897
function main() {
12998

99+
echo "::group::stop_docker"
130100
stop_docker
101+
echo "::endgroup::"
102+
103+
echo "::group::build_docker_images"
131104
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
105+
echo "::endgroup::"
106+
107+
echo "::group::start_services"
132108
start_services
109+
echo "::endgroup::"
133110

111+
echo "::group::validate_megaservice"
134112
validate_megaservice
135-
# validate_frontend
113+
echo "::endgroup::"
136114

115+
echo "::group::stop_docker"
137116
stop_docker
138-
echo y | docker system prune
117+
docker system prune -f
118+
echo "::endgroup::"
139119

140120
}
141121

0 commit comments

Comments
 (0)