From 48ddd4cc868bb8e970c7b67bf7001e83fc1bbd60 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 13 Jun 2025 16:15:07 +0800 Subject: [PATCH 01/14] Fix SC1082 Signed-off-by: ZePan110 --- .github/workflows/mix-code-scan.yml | 32 +++++++++++++++++++ tests/chathistory/test_chathistory_mongo.sh | 2 +- tests/llms/test_llms_faq-generation_tgi.sh | 2 +- ...st_llms_faq-generation_tgi_on_intel_hpu.sh | 2 +- tests/llms/test_llms_faq-generation_vllm.sh | 2 +- ...t_llms_faq-generation_vllm_on_intel_hpu.sh | 2 +- 6 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mix-code-scan.yml b/.github/workflows/mix-code-scan.yml index c04c02a49c..37ce7d7840 100644 --- a/.github/workflows/mix-code-scan.yml +++ b/.github/workflows/mix-code-scan.yml @@ -64,3 +64,35 @@ jobs: with: name: ${{ matrix.job_name }} path: ${{ github.workspace }}/.github/workflows/scripts/codeScan/${{ matrix.job_name }}.* + + shell-check: + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read + contents: read + steps: + - name: Repository checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get changed shell files + id: get-changed-files + run: | + base_commit="${{ github.event.pull_request.base.sha }}" + merged_commit="${{ github.sha }}" + echo "base_commit=${base_commit}" >> $GITHUB_ENV + echo "merged_commit=${merged_commit}" >> $GITHUB_ENV + changed_files="$(git diff --name-only --diff-filter=AM ${base_commit} ${merged_commit} | grep -E '\.sh$' | tr '\n' ' ')" || true + echo "changed_files=${changed_files}" + echo "changed_files=${changed_files}" >> $GITHUB_ENV + + - name: ShellCheck + if: ${{ env.changed_files != '' }} + uses: ludeeus/action-shellcheck@2.0.0 + # env: + # SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 + with: + path: ${{ env.changed_files }} + severity: warning diff --git a/tests/chathistory/test_chathistory_mongo.sh b/tests/chathistory/test_chathistory_mongo.sh index 4bb098d79c..3fa5bd443c 100644 --- a/tests/chathistory/test_chathistory_mongo.sh +++ b/tests/chathistory/test_chathistory_mongo.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/llms/test_llms_faq-generation_tgi.sh b/tests/llms/test_llms_faq-generation_tgi.sh index 68dd5b2243..836a717757 100644 --- a/tests/llms/test_llms_faq-generation_tgi.sh +++ b/tests/llms/test_llms_faq-generation_tgi.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/llms/test_llms_faq-generation_tgi_on_intel_hpu.sh b/tests/llms/test_llms_faq-generation_tgi_on_intel_hpu.sh index 8e14d51dd5..8637362b23 100644 --- a/tests/llms/test_llms_faq-generation_tgi_on_intel_hpu.sh +++ b/tests/llms/test_llms_faq-generation_tgi_on_intel_hpu.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/llms/test_llms_faq-generation_vllm.sh b/tests/llms/test_llms_faq-generation_vllm.sh index 04ca9570aa..f200c1e644 100644 --- a/tests/llms/test_llms_faq-generation_vllm.sh +++ b/tests/llms/test_llms_faq-generation_vllm.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/llms/test_llms_faq-generation_vllm_on_intel_hpu.sh b/tests/llms/test_llms_faq-generation_vllm_on_intel_hpu.sh index c875fa70ae..3e03a4009b 100644 --- a/tests/llms/test_llms_faq-generation_vllm_on_intel_hpu.sh +++ b/tests/llms/test_llms_faq-generation_vllm_on_intel_hpu.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 From 715092cd2dc3d3f331d2c9a1cd54ddda18f90e6d Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 13 Jun 2025 16:17:32 +0800 Subject: [PATCH 02/14] Fix SC1128 Signed-off-by: ZePan110 --- comps/text2kg/src/environment_setup.sh | 2 +- comps/third_parties/gpt-sovits/src/start.sh | 4 ++-- .../nginx/deployment/docker_compose/start-nginx.sh | 2 +- tests/third_parties/test_third_parties_nebula_docker.sh | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/comps/text2kg/src/environment_setup.sh b/comps/text2kg/src/environment_setup.sh index 293af190d0..c319af2629 100644 --- a/comps/text2kg/src/environment_setup.sh +++ b/comps/text2kg/src/environment_setup.sh @@ -1,6 +1,6 @@ +#!/bin/bash # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -#!/bin/bash ####################################################################### # Proxy ####################################################################### diff --git a/comps/third_parties/gpt-sovits/src/start.sh b/comps/third_parties/gpt-sovits/src/start.sh index 5f2db68d75..412056f388 100644 --- a/comps/third_parties/gpt-sovits/src/start.sh +++ b/comps/third_parties/gpt-sovits/src/start.sh @@ -1,8 +1,8 @@ +#!/bin/bash + # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -#!/bin/bash - # Download default voice reference wget "https://github.com/intel/intel-extension-for-transformers/raw/refs/heads/main/intel_extension_for_transformers/neural_chat/assets/audio/welcome_cn.wav" diff --git a/comps/third_parties/nginx/deployment/docker_compose/start-nginx.sh b/comps/third_parties/nginx/deployment/docker_compose/start-nginx.sh index ed3aef76af..4e831b4f6c 100644 --- a/comps/third_parties/nginx/deployment/docker_compose/start-nginx.sh +++ b/comps/third_parties/nginx/deployment/docker_compose/start-nginx.sh @@ -1,6 +1,6 @@ +#!/bin/sh # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -#!/bin/sh envsubst '${FRONTEND_SERVICE_IP} ${FRONTEND_SERVICE_PORT} ${BACKEND_SERVICE_NAME} ${BACKEND_SERVICE_IP} ${BACKEND_SERVICE_PORT} ${DATAPREP_SERVICE_IP} ${DATAPREP_SERVICE_PORT}' < /etc/nginx/nginx.conf.template > /etc/nginx/conf.d/default.conf nginx -g 'daemon off;' diff --git a/tests/third_parties/test_third_parties_nebula_docker.sh b/tests/third_parties/test_third_parties_nebula_docker.sh index 39a9145c7e..ff9868ce22 100755 --- a/tests/third_parties/test_third_parties_nebula_docker.sh +++ b/tests/third_parties/test_third_parties_nebula_docker.sh @@ -1,8 +1,8 @@ +#!/bin/bash + # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -#!/bin/bash - set -euo pipefail set -x From 748707537705d7bb0ef9ab53f45491203306c147 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 13 Jun 2025 16:29:58 +0800 Subject: [PATCH 03/14] Fix SC2069 Signed-off-by: ZePan110 --- tests/agent/test_agent_langchain_on_intel_hpu.sh | 2 +- tests/guardrails/test_guardrails_bias_detection_distilbias.sh | 2 +- .../test_guardrails_guardrails_llamaguard_on_intel_hpu.sh | 2 +- .../test_guardrails_guardrails_wildguard_on_intel_hpu.sh | 2 +- .../test_guardrails_hallucination_detection_on_intel_hpu.sh | 2 +- .../guardrails/test_guardrails_prompt_injection_promptguard.sh | 2 +- .../test_guardrails_toxicity_detection_toxicdetection.sh | 2 +- tests/third_parties/_test_third_parties_sglang.sh | 2 +- tests/third_parties/test_third_parties_ipex.sh | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/agent/test_agent_langchain_on_intel_hpu.sh b/tests/agent/test_agent_langchain_on_intel_hpu.sh index ceef5ee27b..da27833b4e 100644 --- a/tests/agent/test_agent_langchain_on_intel_hpu.sh +++ b/tests/agent/test_agent_langchain_on_intel_hpu.sh @@ -453,7 +453,7 @@ function main() { stop_docker - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } main diff --git a/tests/guardrails/test_guardrails_bias_detection_distilbias.sh b/tests/guardrails/test_guardrails_bias_detection_distilbias.sh index e47ecce238..90cb2f574f 100644 --- a/tests/guardrails/test_guardrails_bias_detection_distilbias.sh +++ b/tests/guardrails/test_guardrails_bias_detection_distilbias.sh @@ -70,7 +70,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh b/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh index da77898a13..ec568eebf9 100644 --- a/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh +++ b/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh @@ -80,7 +80,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh b/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh index 054ac861a7..f3ad130965 100644 --- a/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh +++ b/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh @@ -79,7 +79,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_hallucination_detection_on_intel_hpu.sh b/tests/guardrails/test_guardrails_hallucination_detection_on_intel_hpu.sh index 34abccf7a1..eb99fddc34 100644 --- a/tests/guardrails/test_guardrails_hallucination_detection_on_intel_hpu.sh +++ b/tests/guardrails/test_guardrails_hallucination_detection_on_intel_hpu.sh @@ -115,7 +115,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_prompt_injection_promptguard.sh b/tests/guardrails/test_guardrails_prompt_injection_promptguard.sh index c6bd20038a..03e84c128f 100644 --- a/tests/guardrails/test_guardrails_prompt_injection_promptguard.sh +++ b/tests/guardrails/test_guardrails_prompt_injection_promptguard.sh @@ -112,7 +112,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_toxicity_detection_toxicdetection.sh b/tests/guardrails/test_guardrails_toxicity_detection_toxicdetection.sh index cb172bbd75..905b27847c 100644 --- a/tests/guardrails/test_guardrails_toxicity_detection_toxicdetection.sh +++ b/tests/guardrails/test_guardrails_toxicity_detection_toxicdetection.sh @@ -66,7 +66,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/third_parties/_test_third_parties_sglang.sh b/tests/third_parties/_test_third_parties_sglang.sh index 4a97c88ac7..28eb0004b1 100644 --- a/tests/third_parties/_test_third_parties_sglang.sh +++ b/tests/third_parties/_test_third_parties_sglang.sh @@ -77,7 +77,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/third_parties/test_third_parties_ipex.sh b/tests/third_parties/test_third_parties_ipex.sh index e172148c33..7bce0f0692 100644 --- a/tests/third_parties/test_third_parties_ipex.sh +++ b/tests/third_parties/test_third_parties_ipex.sh @@ -71,7 +71,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } From 4fe780f66caa91e9fd1522b63adff65749935485 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 13 Jun 2025 16:33:56 +0800 Subject: [PATCH 04/14] Fix SC2064 Signed-off-by: ZePan110 --- tests/utils/wait-for-it.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/wait-for-it.sh b/tests/utils/wait-for-it.sh index 04a81c3039..fe437aeede 100755 --- a/tests/utils/wait-for-it.sh +++ b/tests/utils/wait-for-it.sh @@ -61,7 +61,7 @@ wait_for_wrapper() timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & fi WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT + trap 'kill -INT -$WAITFORIT_PID' INT wait $WAITFORIT_PID WAITFORIT_RESULT=$? if [[ $WAITFORIT_RESULT -ne 0 ]]; then From 1758ae0a7bfe9a4a8197497809c4ff09431d9262 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 13 Jun 2025 16:51:11 +0800 Subject: [PATCH 05/14] Fix SC2148 Signed-off-by: ZePan110 --- comps/animation/src/docker_run.sh | 1 + comps/finetuning/src/launch.sh | 1 + .../build_docker_hallucination_microservice.sh | 1 + .../hallucination_detection/launch_hallucination_microservice.sh | 1 + comps/struct2graph/src/environment_setup.sh | 1 + comps/text2graph/src/setup_service_env.sh | 1 + comps/third_parties/video-llama/src/start.sh | 1 + comps/third_parties/wav2lip/src/download_ckpts.sh | 1 + tests/agent/build_vllm_gaudi.sh | 1 + tests/agent/launch_vllm_gaudi.sh | 1 + tests/agent/sql_agent_test/run_data_split.sh | 1 + tests/llms/llm_utils.sh | 1 + 12 files changed, 12 insertions(+) diff --git a/comps/animation/src/docker_run.sh b/comps/animation/src/docker_run.sh index 286ed3bc42..2384c12713 100644 --- a/comps/animation/src/docker_run.sh +++ b/comps/animation/src/docker_run.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/finetuning/src/launch.sh b/comps/finetuning/src/launch.sh index 43730c38ef..b0eabe10d4 100644 --- a/comps/finetuning/src/launch.sh +++ b/comps/finetuning/src/launch.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/guardrails/src/hallucination_detection/build_docker_hallucination_microservice.sh b/comps/guardrails/src/hallucination_detection/build_docker_hallucination_microservice.sh index df4139110b..d11af098a5 100644 --- a/comps/guardrails/src/hallucination_detection/build_docker_hallucination_microservice.sh +++ b/comps/guardrails/src/hallucination_detection/build_docker_hallucination_microservice.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/guardrails/src/hallucination_detection/launch_hallucination_microservice.sh b/comps/guardrails/src/hallucination_detection/launch_hallucination_microservice.sh index 0f6d56956c..c5d4143119 100644 --- a/comps/guardrails/src/hallucination_detection/launch_hallucination_microservice.sh +++ b/comps/guardrails/src/hallucination_detection/launch_hallucination_microservice.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/struct2graph/src/environment_setup.sh b/comps/struct2graph/src/environment_setup.sh index edc3981c89..215015cefa 100644 --- a/comps/struct2graph/src/environment_setup.sh +++ b/comps/struct2graph/src/environment_setup.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/text2graph/src/setup_service_env.sh b/comps/text2graph/src/setup_service_env.sh index b6ec4cf133..518902abf6 100755 --- a/comps/text2graph/src/setup_service_env.sh +++ b/comps/text2graph/src/setup_service_env.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/third_parties/video-llama/src/start.sh b/comps/third_parties/video-llama/src/start.sh index f016ad1a88..de346590de 100644 --- a/comps/third_parties/video-llama/src/start.sh +++ b/comps/third_parties/video-llama/src/start.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/third_parties/wav2lip/src/download_ckpts.sh b/comps/third_parties/wav2lip/src/download_ckpts.sh index a729c8ffd6..fd4ce465f1 100644 --- a/comps/third_parties/wav2lip/src/download_ckpts.sh +++ b/comps/third_parties/wav2lip/src/download_ckpts.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/agent/build_vllm_gaudi.sh b/tests/agent/build_vllm_gaudi.sh index 71b24dc0f3..90bb70cebe 100644 --- a/tests/agent/build_vllm_gaudi.sh +++ b/tests/agent/build_vllm_gaudi.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/agent/launch_vllm_gaudi.sh b/tests/agent/launch_vllm_gaudi.sh index 3ccb60a96d..0eca77cd19 100644 --- a/tests/agent/launch_vllm_gaudi.sh +++ b/tests/agent/launch_vllm_gaudi.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/agent/sql_agent_test/run_data_split.sh b/tests/agent/sql_agent_test/run_data_split.sh index 2fc2dfcb0e..07f1d71474 100644 --- a/tests/agent/sql_agent_test/run_data_split.sh +++ b/tests/agent/sql_agent_test/run_data_split.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/llms/llm_utils.sh b/tests/llms/llm_utils.sh index b86f8283aa..d06470d6be 100644 --- a/tests/llms/llm_utils.sh +++ b/tests/llms/llm_utils.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 From 81256b9b4f766b996f16217b3577e336ef72790e Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 13 Jun 2025 17:08:05 +0800 Subject: [PATCH 06/14] Fix SC1007 Signed-off-by: ZePan110 --- .../test_guardrails_guardrails_llamaguard_on_intel_hpu.sh | 4 ++-- .../test_guardrails_guardrails_wildguard_on_intel_hpu.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh b/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh index ec568eebf9..bdf65346ca 100644 --- a/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh +++ b/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh @@ -41,7 +41,7 @@ function start_service() { function validate_microservice() { echo "Validate microservice started" echo "test 1 - violated policies" - result=$(http_proxy= curl http://localhost:11303/v1/guardrails -X POST -d '{"text":"How do you buy a tiger in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') + result=$(http_proxy="" curl http://localhost:11303/v1/guardrails -X POST -d '{"text":"How do you buy a tiger in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') if [[ $result == *"Violated"* ]]; then echo "Result correct." else @@ -51,7 +51,7 @@ function validate_microservice() { exit 1 fi echo "test 2 - safe" - result=$(http_proxy= curl http://localhost:11303/v1/guardrails -X POST -d '{"text":"How do you buy a car in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') + result=$(http_proxy="" curl http://localhost:11303/v1/guardrails -X POST -d '{"text":"How do you buy a car in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') if [[ $result == *"car"* ]]; then echo "Result correct." else diff --git a/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh b/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh index f3ad130965..34a674bfaa 100644 --- a/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh +++ b/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh @@ -41,7 +41,7 @@ function start_service() { function validate_microservice() { echo "Validate microservice started" echo "test 1 - violated policies" - result=$(http_proxy= curl http://localhost:11304/v1/guardrails -X POST -d '{"text":"How do you buy a tiger in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') + result=$(http_proxy="" curl http://localhost:11304/v1/guardrails -X POST -d '{"text":"How do you buy a tiger in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') if [[ $result == *"Violated"* ]]; then echo "Result correct." else @@ -51,7 +51,7 @@ function validate_microservice() { exit 1 fi echo "test 2 - safe" - result=$(http_proxy= curl http://localhost:11304/v1/guardrails -X POST -d '{"text":"How do you buy a car in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') + result=$(http_proxy="" curl http://localhost:11304/v1/guardrails -X POST -d '{"text":"How do you buy a car in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') if [[ $result == *"car"* ]]; then echo "Result correct." else From 183f1272217978fcab27584eea875668d6fc54ba Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Mon, 16 Jun 2025 09:33:12 +0800 Subject: [PATCH 07/14] Fix SC1083 Signed-off-by: ZePan110 --- .github/workflows/mix-code-scan.yml | 2 +- comps/text2kg/src/environment_setup.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mix-code-scan.yml b/.github/workflows/mix-code-scan.yml index 37ce7d7840..cad4ceeebc 100644 --- a/.github/workflows/mix-code-scan.yml +++ b/.github/workflows/mix-code-scan.yml @@ -92,7 +92,7 @@ jobs: if: ${{ env.changed_files != '' }} uses: ludeeus/action-shellcheck@2.0.0 # env: - # SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 + # SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 -e SC2206 -e SC2128 -e SC1078 -e SC2068 with: path: ${{ env.changed_files }} severity: warning diff --git a/comps/text2kg/src/environment_setup.sh b/comps/text2kg/src/environment_setup.sh index c319af2629..0fc9c1142d 100644 --- a/comps/text2kg/src/environment_setup.sh +++ b/comps/text2kg/src/environment_setup.sh @@ -26,8 +26,8 @@ export NEO4J_USERNAME=${NEO4J_USERNAME:-"neo4j"} export NEO4J_PASSWORD=${NEO4J_PASSWORD:-"neo4j_password"} export NEO4J_URL=${NEO4J_URL:-"neo4j://localhost:7687"} export NEO4J_URI=${NEO4J_URI:-"neo4j://localhost:7687"} -export NEO4J_PORT1={$NEO4J_PORT1:-7474}:7474 -export NEO4J_PORT2={$NEO4J_PORT2:-7687}:7687 +export NEO4J_PORT1=${NEO4J_PORT1:-7474}:7474 +export NEO4J_PORT2=${NEO4J_PORT2:-7687}:7687 export NEO4J_AUTH=neo4j/password export NEO4J_PLUGINS=\[\"apoc\"\] export NEO4J_HEALTH_URL="http://localhost:7474" From 9d95158f4298f301b507c52d172cd3de4bba47cd Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Mon, 16 Jun 2025 09:45:52 +0800 Subject: [PATCH 08/14] Fix SC2038,SC1010 Signed-off-by: ZePan110 --- .github/workflows/scripts/freeze_images.sh | 6 +++--- .github/workflows/scripts/freeze_requirements.sh | 4 ++-- comps/finetuning/src/integrations/xtune/prepare_xtune.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scripts/freeze_images.sh b/.github/workflows/scripts/freeze_images.sh index 2e3e0787d5..fbf175326f 100644 --- a/.github/workflows/scripts/freeze_images.sh +++ b/.github/workflows/scripts/freeze_images.sh @@ -24,9 +24,9 @@ function replace_image_version() { echo "version is empty" else echo "replace $repo_image:latest with $repo_image:$version" - find . -name "Dockerfile*" | xargs sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" - find . -name "*.yaml" | xargs sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" - find . -name "*.md" | xargs sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" + find . -name "Dockerfile*" -print0 | xargs -0 sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" + find . -name "*.yaml" -print0 | xargs -0 sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" + find . -name "*.md" -print0 | xargs -0 sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" fi } diff --git a/.github/workflows/scripts/freeze_requirements.sh b/.github/workflows/scripts/freeze_requirements.sh index d4b6a8633a..4ff2e982cf 100644 --- a/.github/workflows/scripts/freeze_requirements.sh +++ b/.github/workflows/scripts/freeze_requirements.sh @@ -57,8 +57,8 @@ function main() { check_branch_name echo "::group::pip install --no-cache-dir pip-tools" && pip install --no-cache-dir pip-tools --upgrade && echo "::endgroup::" export -f freeze - find . -name "requirements.txt" | xargs -n 1 -I {} bash -c 'freeze "$@"' _ {} - find . -name "requirements-runtime.txt" | xargs -n 1 -I {} bash -c 'freeze "$@"' _ {} + find . -name "requirements.txt" -print0 | xargs -0 -n 1 -I {} bash -c 'freeze "$@"' _ {} + find . -name "requirements-runtime.txt" -print0 | xargs -0 -n 1 -I {} bash -c 'freeze "$@"' _ {} } main diff --git a/comps/finetuning/src/integrations/xtune/prepare_xtune.sh b/comps/finetuning/src/integrations/xtune/prepare_xtune.sh index f35ca70d43..2fa24102ac 100644 --- a/comps/finetuning/src/integrations/xtune/prepare_xtune.sh +++ b/comps/finetuning/src/integrations/xtune/prepare_xtune.sh @@ -52,6 +52,6 @@ else ps aux | grep "llamafactory-cli webui" | head -n 1 | awk '{print $2}' |xargs kill echo "kill webui service done" fi - echo 0 >> done + echo 0 >> "done" echo "Please follow README.md to install driver or update torch lib" fi From 03901f8842cb069320276ed75e670e2bdb516afd Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Mon, 16 Jun 2025 09:52:01 +0800 Subject: [PATCH 09/14] Fix SC2061 Signed-off-by: ZePan110 --- .github/workflows/scripts/get_test_matrix.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/scripts/get_test_matrix.sh b/.github/workflows/scripts/get_test_matrix.sh index 8b93662796..c810325604 100644 --- a/.github/workflows/scripts/get_test_matrix.sh +++ b/.github/workflows/scripts/get_test_matrix.sh @@ -44,7 +44,7 @@ function find_test_1() { # Accurate matching test scripts # find_test=$(find ./tests -type f \( -name test_${service_name}_${integrations}.sh -o -name test_${service_name}_${integrations}_on_*.sh \)) || true # Fuzzy matching test scripts, for example, llms/src/text-generation/integrations/opea.py match several tests. - find_test=$(find ./tests -type f -name test_${service_name}_${integration}*.sh) || true + find_test=$(find ./tests -type f -name "test_${service_name}_${integration}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" else @@ -54,7 +54,7 @@ function find_test_1() { done fi if [ "$run_all_interation" = "true" ]; then - find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true + find_test=$(find ./tests -type f -name "test_${service_name}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" fi @@ -62,7 +62,7 @@ function find_test_1() { elif [[ $(echo ${service_path} | grep "third_parties") ]]; then # new org with `src` and `third_parties` folder service_name=$(echo $service_path | sed 's:/src::' | tr '/' '_' | cut -c7-) # comps/third_parties/vllm/src -> third_parties_vllm - find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true + find_test=$(find ./tests -type f -name "test_${service_name}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" fi @@ -77,7 +77,7 @@ function find_test_1() { else # old org without 'src' folder service_name=$(echo $service_path | tr '/' '_' | cut -c7-) # comps/retrievers/redis/langchain -> retrievers_redis_langchain - find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true + find_test=$(find ./tests -type f -name "test_${service_name}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" fi @@ -140,7 +140,7 @@ function find_test_3() { _domain=${_domain#compose_} _service=${_service}_${_domain} fi - find_test=$(find ./tests -type f -name test_${_service}*.sh) || true + find_test=$(find ./tests -type f -name "test_${_service}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" fi From 49fa7351da21b0dc7fd8e3c9c27885c929510a4e Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Mon, 16 Jun 2025 09:57:35 +0800 Subject: [PATCH 10/14] Fix SC2156 Signed-off-by: ZePan110 --- .github/workflows/mix-code-scan.yml | 2 +- .github/workflows/scripts/test_ut.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mix-code-scan.yml b/.github/workflows/mix-code-scan.yml index cad4ceeebc..9a12bf85f6 100644 --- a/.github/workflows/mix-code-scan.yml +++ b/.github/workflows/mix-code-scan.yml @@ -92,7 +92,7 @@ jobs: if: ${{ env.changed_files != '' }} uses: ludeeus/action-shellcheck@2.0.0 # env: - # SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 -e SC2206 -e SC2128 -e SC1078 -e SC2068 + # SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 -e SC2206 -e SC2128 -e SC1078 -e SC2068 -e SC2089 -e SC2090 -e SC2140 -e SC2010 -e SC2061 with: path: ${{ env.changed_files }} severity: warning diff --git a/.github/workflows/scripts/test_ut.sh b/.github/workflows/scripts/test_ut.sh index 5f70c9a60e..39ce87f607 100644 --- a/.github/workflows/scripts/test_ut.sh +++ b/.github/workflows/scripts/test_ut.sh @@ -10,7 +10,7 @@ cd /GenAIComps/tests if [ $test_name = 'mega' ]; then echo "run mega test" cd cores - find . -name "*.yaml" -exec sh -c 'x="{}"; cp $x ./' \; + find . -name "*.yaml" -exec sh -c 'cp "$1" ./ ' _ {} \; comps_path=$(python -c 'import comps; print(comps.__path__[0])') find . -name "test*.py" | sed "s,\.\/,python -m pytest --cov=\"${comps_path}\" --cov-report term --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run.sh bash run.sh 2>&1 | tee ${ut_log_name} From cbc4640ba127f56b9b2bef8ec46861064f9c10c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 02:01:34 +0000 Subject: [PATCH 11/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/third_parties/_test_third_parties_sglang.sh | 2 +- tests/third_parties/test_third_parties_ipex.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/third_parties/_test_third_parties_sglang.sh b/tests/third_parties/_test_third_parties_sglang.sh index 28eb0004b1..170f7488f2 100644 --- a/tests/third_parties/_test_third_parties_sglang.sh +++ b/tests/third_parties/_test_third_parties_sglang.sh @@ -77,7 +77,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune > /dev/null 2>&1 + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/third_parties/test_third_parties_ipex.sh b/tests/third_parties/test_third_parties_ipex.sh index 7bce0f0692..b5dd9cfa8d 100644 --- a/tests/third_parties/test_third_parties_ipex.sh +++ b/tests/third_parties/test_third_parties_ipex.sh @@ -71,7 +71,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune > /dev/null 2>&1 + echo y | docker system prune > /dev/null 2>&1 } From 835d1f2b331edda98014b467723079b2190d11a3 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Mon, 16 Jun 2025 10:35:14 +0800 Subject: [PATCH 12/14] Add exclude Signed-off-by: ZePan110 --- .github/workflows/mix-code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mix-code-scan.yml b/.github/workflows/mix-code-scan.yml index 9a12bf85f6..4c94a18277 100644 --- a/.github/workflows/mix-code-scan.yml +++ b/.github/workflows/mix-code-scan.yml @@ -91,8 +91,8 @@ jobs: - name: ShellCheck if: ${{ env.changed_files != '' }} uses: ludeeus/action-shellcheck@2.0.0 - # env: - # SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 -e SC2206 -e SC2128 -e SC1078 -e SC2068 -e SC2089 -e SC2090 -e SC2140 -e SC2010 -e SC2061 + env: + SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 -e SC2206 -e SC2128 -e SC1078 -e SC2068 -e SC2089 -e SC2090 -e SC2140 -e SC2010 -e SC2061 with: path: ${{ env.changed_files }} severity: warning From 3af7c0de7a5c50bd9bc654d1c5153f3324870ab2 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Mon, 16 Jun 2025 10:50:48 +0800 Subject: [PATCH 13/14] Fix SC2069,SC2260 Signed-off-by: ZePan110 --- .github/workflows/mix-code-scan.yml | 2 +- tests/guardrails/test_guardrails_pii_detection_piidetection.sh | 2 +- tests/utils/wait-for-it.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mix-code-scan.yml b/.github/workflows/mix-code-scan.yml index 4c94a18277..8b49969e0c 100644 --- a/.github/workflows/mix-code-scan.yml +++ b/.github/workflows/mix-code-scan.yml @@ -92,7 +92,7 @@ jobs: if: ${{ env.changed_files != '' }} uses: ludeeus/action-shellcheck@2.0.0 env: - SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 -e SC2206 -e SC2128 -e SC1078 -e SC2068 -e SC2089 -e SC2090 -e SC2140 -e SC2010 -e SC2061 + SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 -e SC2206 -e SC2128 -e SC1078 -e SC2068 -e SC2089 -e SC2090 -e SC2140 -e SC2010 -e SC2061 -e SC2309 with: path: ${{ env.changed_files }} severity: warning diff --git a/tests/guardrails/test_guardrails_pii_detection_piidetection.sh b/tests/guardrails/test_guardrails_pii_detection_piidetection.sh index ae0f7dc0b3..914000b323 100644 --- a/tests/guardrails/test_guardrails_pii_detection_piidetection.sh +++ b/tests/guardrails/test_guardrails_pii_detection_piidetection.sh @@ -66,7 +66,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/utils/wait-for-it.sh b/tests/utils/wait-for-it.sh index fe437aeede..8db49e075c 100755 --- a/tests/utils/wait-for-it.sh +++ b/tests/utils/wait-for-it.sh @@ -153,7 +153,7 @@ if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then WAITFORIT_ISBUSY=1 # Check if busybox timeout uses -t flag # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then + if timeout 2>&1 | tee /dev/stdout | grep -q -e '-t '; then WAITFORIT_BUSYTIMEFLAG="-t" fi else From 13c284acd124087c382a381f13e792d8722a013a Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Jun 2025 08:39:58 +0800 Subject: [PATCH 14/14] Delete tests/guardrails/test_guardrails_pii_detection_piidetection.sh --- ...t_guardrails_pii_detection_piidetection.sh | 73 ------------------- 1 file changed, 73 deletions(-) delete mode 100644 tests/guardrails/test_guardrails_pii_detection_piidetection.sh diff --git a/tests/guardrails/test_guardrails_pii_detection_piidetection.sh b/tests/guardrails/test_guardrails_pii_detection_piidetection.sh deleted file mode 100644 index 914000b323..0000000000 --- a/tests/guardrails/test_guardrails_pii_detection_piidetection.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Copyright (C) 2025 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -set -x - -WORKPATH=$(dirname "$PWD") -ip_address=$(hostname -I | awk '{print $1}') - -function build_docker_images() { - echo "Start building docker images for microservice" - cd $WORKPATH - docker build --no-cache -t opea/guardrails-pii-detection:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/guardrails/src/pii_detection/Dockerfile . - if [ $? -ne 0 ]; then - echo "opea/guardrails-pii-detection built fail" - exit 1 - else - echo "opea/guardrails-pii-detection built successful" - fi -} - -function start_service() { - echo "Starting microservice" - export pii_detection_port=9081 - docker run -d --runtime=runc --name="test-comps-guardrails-pii-detection-endpoint" -p $pii_detection_port:$pii_detection_port --ipc=host -e PII_DETECTION_PORT=$pii_detection_port -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy-$no_proxy opea/guardrails-pii-detection:comps - sleep 25 - echo "Microservice started" -} - -function validate_microservice() { - echo "Validate microservice started" - echo "test 1 - PII" - result=$(curl localhost:9081/v1/pii -X POST -d '{"text":"My name is John Doe and my phone number is (555) 555-5555."}' -H 'Content-Type: application/json') - if [[ $result == *"PERSON"* && $result == *"PHONE_NUMBER"* ]]; then - echo "Result correct." - else - docker logs test-comps-guardrails-pii-detection-endpoint - exit 1 - fi - echo "test 2 - no PII" - result=$(curl localhost:9081/v1/pii -X POST -d '{"text":"How to write a paper on raising dogs?"}' -H 'Content-Type: application/json') - if [[ $result == *"paper"* ]]; then - echo "Result correct." - else - echo "Result wrong." - docker logs test-comps-guardrails-pii-detection-endpoint - exit 1 - fi - echo "Validate microservice completed" -} - -function stop_docker() { - cid=$(docker ps -aq --filter "name=test-comps-guardrails-pii-detection-endpoint") - echo "Shutdown legacy containers "$cid - if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi -} - -function main() { - - stop_docker - - build_docker_images - start_service - - validate_microservice - - stop_docker - echo "cleanup container images and volumes" - echo y | docker system prune > /dev/null 2>&1 - -} - -main