diff --git a/.github/workflows/docker/compose/dataprep-compose.yaml b/.github/workflows/docker/compose/dataprep-compose.yaml index 893f6947ea..39d8f9e618 100644 --- a/.github/workflows/docker/compose/dataprep-compose.yaml +++ b/.github/workflows/docker/compose/dataprep-compose.yaml @@ -7,3 +7,7 @@ services: build: dockerfile: comps/dataprep/src/Dockerfile image: ${REGISTRY:-opea}/dataprep:${TAG:-latest} + dataprep-openeuler: + build: + dockerfile: comps/dataprep/src/Dockerfile.openEuler + image: ${REGISTRY:-opea}/dataprep:${TAG:-latest}-openeuler diff --git a/.github/workflows/docker/compose/embeddings-compose.yaml b/.github/workflows/docker/compose/embeddings-compose.yaml index 13c1fd5a2d..e6f762a486 100644 --- a/.github/workflows/docker/compose/embeddings-compose.yaml +++ b/.github/workflows/docker/compose/embeddings-compose.yaml @@ -7,3 +7,7 @@ services: build: dockerfile: comps/embeddings/src/Dockerfile image: ${REGISTRY:-opea}/embedding:${TAG:-latest} + embedding-openeuler: + build: + dockerfile: comps/embeddings/src/Dockerfile.openEuler + image: ${REGISTRY:-opea}/embedding:${TAG:-latest}-openeuler diff --git a/.github/workflows/docker/compose/rerankings-compose.yaml b/.github/workflows/docker/compose/rerankings-compose.yaml index 66e878af89..4ed88b7cc7 100644 --- a/.github/workflows/docker/compose/rerankings-compose.yaml +++ b/.github/workflows/docker/compose/rerankings-compose.yaml @@ -7,3 +7,7 @@ services: build: dockerfile: comps/rerankings/src/Dockerfile image: ${REGISTRY:-opea}/reranking:${TAG:-latest} + reranking-openeuler: + build: + dockerfile: comps/rerankings/src/Dockerfile.openEuler + image: ${REGISTRY:-opea}/reranking:${TAG:-latest}-openeuler diff --git a/.github/workflows/docker/compose/retrievers-compose.yaml b/.github/workflows/docker/compose/retrievers-compose.yaml index 72ce523367..718d881494 100644 --- a/.github/workflows/docker/compose/retrievers-compose.yaml +++ b/.github/workflows/docker/compose/retrievers-compose.yaml @@ -7,3 +7,7 @@ services: build: dockerfile: comps/retrievers/src/Dockerfile image: ${REGISTRY:-opea}/retriever:${TAG:-latest} + retriever-openeuler: + build: + dockerfile: comps/retrievers/src/Dockerfile.openEuler + image: ${REGISTRY:-opea}/retriever:${TAG:-latest}-openeuler diff --git a/.github/workflows/docker/compose/third_parties-compose.yaml b/.github/workflows/docker/compose/third_parties-compose.yaml index 0710e8562e..73ed712105 100644 --- a/.github/workflows/docker/compose/third_parties-compose.yaml +++ b/.github/workflows/docker/compose/third_parties-compose.yaml @@ -6,6 +6,10 @@ services: build: dockerfile: comps/third_parties/nginx/src/Dockerfile image: ${REGISTRY:-opea}/nginx:${TAG:-latest} + nginx-openeuler: + build: + dockerfile: comps/third_parties/nginx/src/Dockerfile.openEuler + image: ${REGISTRY:-opea}/nginx:${TAG:-latest}-openeuler embedding-multimodal-clip: build: dockerfile: comps/third_parties/clip/src/Dockerfile diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 14a3b81dc9..59200d4073 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -167,7 +167,7 @@ jobs: consistency="TRUE" yamls=$(find .github/workflows/docker/compose/ -name '*-compose.yaml') for build_yaml in $yamls; do - message=$(python3 .github/workflows/scripts/check-name-agreement.py "$build_yaml") + message=$(python3 .github/workflows/scripts/check_name_agreement.py "$build_yaml") if [[ "$message" != *"consistent"* ]]; then consistency="FALSE" echo "Inconsistent service name and image name found in file $build_yaml." diff --git a/.github/workflows/scripts/check-name-agreement.py b/.github/workflows/scripts/check_name_agreement.py similarity index 89% rename from .github/workflows/scripts/check-name-agreement.py rename to .github/workflows/scripts/check_name_agreement.py index d588a81d97..c4aca4e07d 100644 --- a/.github/workflows/scripts/check-name-agreement.py +++ b/.github/workflows/scripts/check_name_agreement.py @@ -19,8 +19,10 @@ def check_service_image_consistency(data): image_name = service_details.get("image", "") # Extract the image name part after the last '/' image_name_part = image_name.split("/")[-1].split(":")[0] + # Remove '-openeuler' suffix if it exists + fixed_service_name = service_name.rsplit("-openeuler", 1)[0] # Check if the service name is a substring of the image name part - if service_name not in image_name_part: + if fixed_service_name not in image_name_part: # Get the line number of the service name line_number = service_details.lc.line + 1 inconsistencies.append((service_name, image_name, line_number)) diff --git a/.github/workflows/scripts/codeScan/.hadolint.yaml b/.github/workflows/scripts/codeScan/.hadolint.yaml index f232330c9e..9d81ed756b 100644 --- a/.github/workflows/scripts/codeScan/.hadolint.yaml +++ b/.github/workflows/scripts/codeScan/.hadolint.yaml @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 ignored: + - DL3033 - DL3003 - DL3006 - DL3007 diff --git a/.github/workflows/scripts/get_test_matrix.sh b/.github/workflows/scripts/get_test_matrix.sh index c810325604..9df20767c2 100644 --- a/.github/workflows/scripts/get_test_matrix.sh +++ b/.github/workflows/scripts/get_test_matrix.sh @@ -54,10 +54,23 @@ function find_test_1() { done fi if [ "$run_all_interation" = "true" ]; then + set -x find_test=$(find ./tests -type f -name "test_${service_name}*.sh") || true if [ "$find_test" ]; then - fill_in_matrix "$find_test" + # limit test case. If the changed file only to be Dockerfile.openEuler, then only run the script integrate related test. + check_files_except_openeuler_docker=$(printf '%s\n' "${changed_files[@]}"| grep ${service_path} | grep -vE 'Dockerfile.openEuler' | sort -u) || true + if [ -z "$check_files_except_openeuler_docker" ]; then + for test in ${find_test}; do + find_dockerfile=$(grep -rl "openEuler" $test) || true + if [ "$find_dockerfile" ]; then + fill_in_matrix "$test" + fi + done + else + fill_in_matrix "$find_test" + fi fi + set +x fi elif [[ $(echo ${service_path} | grep "third_parties") ]]; then # new org with `src` and `third_parties` folder diff --git a/Dockerfile.openEuler b/Dockerfile.openEuler new file mode 100644 index 0000000000..aa80ed5486 --- /dev/null +++ b/Dockerfile.openEuler @@ -0,0 +1,38 @@ +# Copyright (C) 2025 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +# Base image for GenAIComps based OPEA Python applications +# Build: docker build -t opea/comps-base:{version}-openeuler -f Dockerfile.openEuler . + +ARG IMAGE_NAME=openeuler/python +ARG IMAGE_TAG=3.11.13-oe2403lts + +FROM ${IMAGE_NAME}:${IMAGE_TAG} AS base + +ENV HOME=/home/user + +RUN yum update -y && \ + yum upgrade -y && \ + yum install -y shadow && \ + yum clean all && \ + rm -rf /var/cache/yum + +RUN useradd -m -s /bin/bash user && \ + mkdir -p $HOME && \ + chown -R user $HOME + +WORKDIR $HOME + +COPY *.toml *.py *.txt *.md LICENSE ./ + +ARG uvpip='uv pip install --system --no-cache-dir' +RUN pip install --no-cache-dir --upgrade pip setuptools uv && \ + $uvpip -r requirements.txt + +COPY comps/ comps/ + +ENV PYTHONPATH=$PYTHONPATH:$HOME + +USER user + +ENTRYPOINT ["sh", "-c", "set && ls -la"] diff --git a/comps/dataprep/src/Dockerfile.openEuler b/comps/dataprep/src/Dockerfile.openEuler new file mode 100644 index 0000000000..5785037b18 --- /dev/null +++ b/comps/dataprep/src/Dockerfile.openEuler @@ -0,0 +1,66 @@ +# Copyright (C) 2025 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +FROM openeuler/python:3.11.13-oe2403lts + +ENV LANG=C.UTF-8 + +ARG ARCH="cpu" + +RUN yum update -y && yum install -y \ + gcc g++ make cmake \ + java-1.8.0-openjdk \ + cairo \ + libxslt \ + mesa-libGL \ + jemalloc-devel \ + mariadb-connector-c-devel-3.3.8 \ + libpq-devel \ + poppler-utils \ + tesseract-tools \ + ffmpeg \ + wget curl \ + shadow && \ + yum clean all && \ + rm -rf /var/cache/yum + +ENV TESSDATA_PREFIX=/usr/share/tesseract/tessdata + +RUN LIBREOFFICE_URL=https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/stable/25.2.4/rpm/x86_64/LibreOffice_25.2.4_Linux_x86-64_rpm.tar.gz && \ + wget $LIBREOFFICE_URL -O /tmp/libreOffice.tar.gz && \ + tar --strip-components=1 -xvf /tmp/libreOffice.tar.gz -C /tmp && \ + yum install -y /tmp/RPMS/*.rpm && \ + yum clean all && \ + rm -fr /tmp/libreOffice.tar.gz /tmp/RPMS && \ + ln -sf /usr/bin/libreoffice25.2 /usr/bin/libreoffice + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +COPY comps /home/user/comps + +ARG uvpip='uv pip install --system --no-cache-dir' +RUN pip install --no-cache-dir --upgrade pip setuptools uv && \ + if [ ${ARCH} = "cpu" ]; then \ + $uvpip torch torchvision --index-url https://download.pytorch.org/whl/cpu; \ + $uvpip -r /home/user/comps/dataprep/src/requirements-cpu.txt; \ + else \ + $uvpip -r /home/user/comps/dataprep/src/requirements-gpu.txt; \ + fi + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +RUN mkdir -p /home/user/comps/dataprep/src/uploaded_files && chown -R user /home/user/comps/dataprep/src/uploaded_files +RUN mkdir -p /data && chown -R user /data + +USER user +ENV NLTK_DATA=/home/user/nltk_data +# air gapped support: predownload all needed nltk data +RUN mkdir -p /home/user/nltk_data && python -m nltk.downloader -d /home/user/nltk_data punkt_tab averaged_perceptron_tagger_eng stopwords +# air gapped support: set model cache dir +ENV HF_HUB_CACHE=/data + +WORKDIR /home/user/comps/dataprep/src + +ENTRYPOINT ["sh", "-c", "python $( [ \"$MULTIMODAL_DATAPREP\" = \"true\" ] && echo 'opea_dataprep_multimodal_microservice.py' || echo 'opea_dataprep_microservice.py')"] diff --git a/comps/embeddings/src/Dockerfile.openEuler b/comps/embeddings/src/Dockerfile.openEuler new file mode 100644 index 0000000000..0dccf4915a --- /dev/null +++ b/comps/embeddings/src/Dockerfile.openEuler @@ -0,0 +1,19 @@ +# Copyright (C) 2025 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +FROM openeuler/python:3.11.13-oe2403lts + +COPY comps /home/comps + +ARG uvpip='uv pip install --system --no-cache-dir' +RUN pip install --no-cache-dir --upgrade pip setuptools uv && \ + $uvpip -r /home/comps/embeddings/src/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home + +WORKDIR /home/comps/embeddings/src/ + +ENV MULTIMODAL_EMBEDDING="false" + +ENTRYPOINT ["sh", "-c", "python $( [ \"$MULTIMODAL_EMBEDDING\" = \"true\" ] && echo 'opea_multimodal_embedding_microservice.py' || echo 'opea_embedding_microservice.py')"] + diff --git a/comps/rerankings/src/Dockerfile.openEuler b/comps/rerankings/src/Dockerfile.openEuler new file mode 100644 index 0000000000..0e6fe858e7 --- /dev/null +++ b/comps/rerankings/src/Dockerfile.openEuler @@ -0,0 +1,40 @@ +# Copyright (C) 2025 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +FROM openeuler/python:3.11.13-oe2403lts + +ENV LANG=C.UTF-8 + +ARG ARCH="cpu" + +RUN yum update -y && yum install -y \ + git \ + mesa-libGL \ + jemalloc-devel \ + shadow && \ + yum clean all && \ + rm -rf /var/cache/yum + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +COPY comps/rerankings/src/requirements* /home/user/comps/rerankings/src/ + +ARG uvpip='uv pip install --system --no-cache-dir' +RUN pip install --no-cache-dir --upgrade pip setuptools uv && \ + if [ ${ARCH} = "cpu" ]; then \ + $uvpip torch --index-url https://download.pytorch.org/whl/cpu; \ + $uvpip -r /home/user/comps/rerankings/src/requirements-cpu.txt; \ + else \ + $uvpip -r /home/user/comps/rerankings/src/requirements-gpu.txt; \ + fi + +COPY comps /home/user/comps +ENV PYTHONPATH=$PYTHONPATH:/home/user + +USER user + +WORKDIR /home/user/comps/rerankings/src + +ENTRYPOINT ["python", "opea_reranking_microservice.py"] diff --git a/comps/retrievers/src/Dockerfile.openEuler b/comps/retrievers/src/Dockerfile.openEuler new file mode 100644 index 0000000000..8182e1d368 --- /dev/null +++ b/comps/retrievers/src/Dockerfile.openEuler @@ -0,0 +1,43 @@ +# Copyright (C) 2025 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +FROM openeuler/python:3.11.13-oe2403lts + +ARG ARCH="cpu" + +RUN yum update -y && yum install -y \ + cairo \ + mesa-libGL \ + glib2 \ + jemalloc-devel \ + mariadb-connector-c-devel-3.3.8 \ + gcc g++ make cmake \ + shadow && \ + yum clean all && \ + rm -rf /var/cache/yum + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +COPY comps /home/user/comps + +ARG uvpip='uv pip install --system --no-cache-dir' +RUN pip install --no-cache-dir --upgrade pip setuptools uv && \ + if [ ${ARCH} = "cpu" ]; then \ + $uvpip torch torchvision --index-url https://download.pytorch.org/whl/cpu; \ + $uvpip -r /home/user/comps/retrievers/src/requirements-cpu.txt; \ + else \ + $uvpip -r /home/user/comps/retrievers/src/requirements-gpu.txt; \ + fi + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +USER user +ENV NLTK_DATA=/home/user/nltk_data +# air gapped support: predownload all needed nltk data +RUN mkdir -p /home/user/nltk_data && python -m nltk.downloader -d /home/user/nltk_data punkt_tab stopwords + +WORKDIR /home/user/comps/retrievers/src + +ENTRYPOINT ["python", "opea_retrievers_microservice.py"] diff --git a/comps/third_parties/nginx/src/Dockerfile.openEuler b/comps/third_parties/nginx/src/Dockerfile.openEuler new file mode 100644 index 0000000000..f06383d659 --- /dev/null +++ b/comps/third_parties/nginx/src/Dockerfile.openEuler @@ -0,0 +1,25 @@ +# Copyright (C) 2025 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + + +FROM openeuler/nginx:1.29.0-oe2403lts + +RUN yum update -y && \ + yum install -y gettext && \ + yum clean all && \ + rm -fr /var/cache/yum + +COPY comps/third_parties/nginx/src/nginx.conf.template /etc/nginx/nginx.conf.template + +ENV FRONTEND_SERVICE_IP=localhost +ENV FRONTEND_SERVICE_PORT=5173 +ENV BACKEND_SERVICE_NAME=chatqna +ENV BACKEND_SERVICE_IP=localhost +ENV BACKEND_SERVICE_PORT=8888 +ENV DATAPREP_SERVICE_IP=localhost +ENV DATAPREP_SERVICE_PORT=6007 + +COPY comps/third_parties/nginx/deployment/docker_compose/start-nginx.sh /usr/local/bin/start-nginx.sh +RUN chmod +x /usr/local/bin/start-nginx.sh + +CMD ["/usr/local/bin/start-nginx.sh"] diff --git a/tests/dataprep/test_dataprep_redis.sh b/tests/dataprep/test_dataprep_redis.sh index 230808a24e..7472ce535f 100644 --- a/tests/dataprep/test_dataprep_redis.sh +++ b/tests/dataprep/test_dataprep_redis.sh @@ -2,7 +2,7 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -set -x +set -xe WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" @@ -18,7 +18,8 @@ source ${SCRIPT_DIR}/dataprep_utils.sh function build_docker_images() { cd $WORKPATH echo $(pwd) - docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile . + dockerfile_name="comps/dataprep/src/$1" + docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" . if [ $? -ne 0 ]; then echo "opea/dataprep built fail" exit 1 @@ -118,11 +119,6 @@ function validate_microservice() { } -function stop_docker() { - cid=$(docker ps -aq --filter "name=dataprep-redis-server*" --filter "name=redis-vector-*" --filter "name=tei-embedding-*") - if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi -} - function stop_service() { cd $WORKPATH/comps/dataprep/deployment/docker_compose/ docker compose down || true @@ -130,9 +126,7 @@ function stop_service() { function main() { - stop_docker - - build_docker_images + build_docker_images "Dockerfile" trap stop_service EXIT echo "Test normal env ..." @@ -148,8 +142,13 @@ function main() { stop_service fi - stop_docker - echo y | docker system prune + echo "Test with openEuler OS ..." + build_docker_images "Dockerfile.openEuler" + start_service + validate_microservice + stop_service + + docker system prune -f } diff --git a/tests/embeddings/test_embeddings_tei.sh b/tests/embeddings/test_embeddings_tei.sh index 5d9fcb911a..f49f9c3d81 100644 --- a/tests/embeddings/test_embeddings_tei.sh +++ b/tests/embeddings/test_embeddings_tei.sh @@ -11,7 +11,8 @@ export DATA_PATH=${model_cache} function build_docker_images() { cd $WORKPATH echo $(pwd) - docker build --no-cache -t opea/embedding:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile . + dockerfile_name="comps/embeddings/src/$1" + docker build --no-cache -t opea/embedding:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" . if [ $? -ne 0 ]; then echo "opea/embedding built fail" exit 1 @@ -72,23 +73,31 @@ function validate_microservice_with_openai() { fi } -function stop_docker() { - cid=$(docker ps -aq --filter "name=tei-embedding-*") - if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi + +function stop_service() { + cd $WORKPATH/comps/embeddings/deployment/docker_compose/ + docker compose down || true } function main() { - stop_docker + build_docker_images "Dockerfile" + trap stop_service EXIT - build_docker_images + echo "Test normal env ..." start_service + validate_microservice + validate_microservice_with_openai + stop_service + echo "Test with openEuler OS ..." + build_docker_images "Dockerfile.openEuler" + start_service validate_microservice validate_microservice_with_openai + stop_service - stop_docker - echo y | docker system prune + docker system prune -f } diff --git a/tests/rerankings/test_rerankings_tei.sh b/tests/rerankings/test_rerankings_tei.sh index 8246b23806..6b8b1014e6 100644 --- a/tests/rerankings/test_rerankings_tei.sh +++ b/tests/rerankings/test_rerankings_tei.sh @@ -11,11 +11,12 @@ export DATA_PATH=${model_cache} function build_docker_images() { cd $WORKPATH + dockerfile_name="comps/rerankings/src/$1" docker build --no-cache \ -t opea/reranking:comps \ --build-arg https_proxy=$https_proxy \ --build-arg http_proxy=$http_proxy \ - -f comps/rerankings/src/Dockerfile . + -f "${dockerfile_name}" . if [ $? -ne 0 ]; then echo "opea/reranking built fail" exit 1 @@ -54,23 +55,28 @@ function validate_microservice() { fi } -function stop_docker() { - cd $WORKPATH/comps/rerankings/deployment/docker_compose - docker compose -f compose.yaml down --remove-orphans +function stop_service() { + cd $WORKPATH/comps/rerankings/deployment/docker_compose/ + docker compose down || true } function main() { - stop_docker + build_docker_images "Dockerfile" + trap stop_service EXIT - build_docker_images + echo "Test normal env ..." start_service - validate_microservice + stop_service - stop_docker - echo y | docker system prune + echo "Test with openEuler OS ..." + build_docker_images "Dockerfile.openEuler" + start_service + validate_microservice + stop_service + docker system prune -f } main diff --git a/tests/retrievers/test_retrievers_redis.sh b/tests/retrievers/test_retrievers_redis.sh index ad8e866c45..c1271d8485 100644 --- a/tests/retrievers/test_retrievers_redis.sh +++ b/tests/retrievers/test_retrievers_redis.sh @@ -18,7 +18,8 @@ service_name_mm="retriever-redis-multimodal" function build_docker_images() { cd $WORKPATH - docker build --no-cache -t ${REGISTRY:-opea}/retriever:${TAG:-latest} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/Dockerfile . + dockerfile_name="comps/retrievers/src/$1" + docker build --no-cache -t ${REGISTRY:-opea}/retriever:${TAG:-latest} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" . if [ $? -ne 0 ]; then echo "opea/retriever built fail" exit 1 @@ -67,8 +68,6 @@ function validate_microservice() { local test_embedding="$1" local container_name="$2" - export PATH="${HOME}/miniforge3/bin:$PATH" - source activate URL="http://${host_ip}:$RETRIEVER_PORT/v1/retrieval" HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "{\"text\":\"test\",\"embedding\":${test_embedding}}" -H 'Content-Type: application/json' "$URL") @@ -94,8 +93,6 @@ function validate_mm_microservice() { local test_embedding="$1" local container_name="$2" - export PATH="${HOME}/miniforge3/bin:$PATH" - source activate URL="http://${host_ip}:$RETRIEVER_PORT/v1/retrieval" # Test the retriever with a b64 image that should be passed through @@ -129,23 +126,30 @@ function validate_mm_microservice() { fi } -function stop_docker() { - cd $WORKPATH/comps/retrievers/deployment/docker_compose - docker compose -f compose.yaml down --remove-orphans + +function stop_service() { + cd $WORKPATH/comps/retrievers/deployment/docker_compose/ + docker compose down || true + cid=$(docker ps -aq --filter "name=redis-vector-db") if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi } function main() { - stop_docker - build_docker_images + build_docker_images "Dockerfile" + trap stop_service EXIT + + export PATH="${HOME}/miniforge3/bin:$PATH" + source activate && sleep 1s + + echo "Test normal env ..." # test text retriever start_service test_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") validate_microservice "$test_embedding" "$service_name" - stop_docker + stop_service # test multimodal retriever start_multimodal_service @@ -154,8 +158,27 @@ function main() { validate_mm_microservice "$test_embedding_multi" "$service_name_mm" # clean env - stop_docker - echo y | docker system prune + stop_service + + echo "Test with openEuler OS ..." + build_docker_images "Dockerfile.openEuler" + # test text retriever + start_service + test_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + validate_microservice "$test_embedding" "$service_name" + stop_service + + # test multimodal retriever + start_multimodal_service + test_embedding_multi=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") + validate_microservice "$test_embedding_multi" "$service_name_mm" + validate_mm_microservice "$test_embedding_multi" "$service_name_mm" + + # clean env + stop_service + + docker system prune -f + } diff --git a/tests/third_parties/test_third_parties_nginx.sh b/tests/third_parties/test_third_parties_nginx.sh index a1d41303e7..99aafd1eed 100644 --- a/tests/third_parties/test_third_parties_nginx.sh +++ b/tests/third_parties/test_third_parties_nginx.sh @@ -10,7 +10,8 @@ ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { cd $WORKPATH - docker build --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -t opea/nginx:comps -f comps/third_parties/nginx/src/Dockerfile . + dockerfile_name="comps/third_parties/nginx/src/$1" + docker build --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -t opea/nginx:comps -f "${dockerfile_name}" . if [ $? -ne 0 ]; then echo "opea/nginx built fail" exit 1 @@ -57,21 +58,28 @@ function validate_service() { fi } -function stop_docker() { +function stop_service() { cid=$(docker ps -aq --filter "name=test-comps-nginx*") if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi } function main() { - stop_docker - build_docker_images - start_service + build_docker_images "Dockerfile" + trap stop_service EXIT + echo "Test normal env ..." + start_service validate_service + stop_service - echo y | docker system prune + echo "Test with openEuler OS ..." + build_docker_images "Dockerfile.openEuler" + start_service + validate_service + stop_service + docker system prune -f } main