diff --git a/.github/workflows/mix-code-scan.yml b/.github/workflows/mix-code-scan.yml index c04c02a49c..8b49969e0c 100644 --- a/.github/workflows/mix-code-scan.yml +++ b/.github/workflows/mix-code-scan.yml @@ -64,3 +64,35 @@ jobs: with: name: ${{ matrix.job_name }} path: ${{ github.workspace }}/.github/workflows/scripts/codeScan/${{ matrix.job_name }}.* + + shell-check: + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read + contents: read + steps: + - name: Repository checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get changed shell files + id: get-changed-files + run: | + base_commit="${{ github.event.pull_request.base.sha }}" + merged_commit="${{ github.sha }}" + echo "base_commit=${base_commit}" >> $GITHUB_ENV + echo "merged_commit=${merged_commit}" >> $GITHUB_ENV + changed_files="$(git diff --name-only --diff-filter=AM ${base_commit} ${merged_commit} | grep -E '\.sh$' | tr '\n' ' ')" || true + echo "changed_files=${changed_files}" + echo "changed_files=${changed_files}" >> $GITHUB_ENV + + - name: ShellCheck + if: ${{ env.changed_files != '' }} + uses: ludeeus/action-shellcheck@2.0.0 + env: + SHELLCHECK_OPTS: -e SC2164 -e SC2154 -e SC2034 -e SC2155 -e SC1090 -e SC2046 -e SC2043 -e SC2206 -e SC2128 -e SC1078 -e SC2068 -e SC2089 -e SC2090 -e SC2140 -e SC2010 -e SC2061 -e SC2309 + with: + path: ${{ env.changed_files }} + severity: warning diff --git a/.github/workflows/scripts/freeze_images.sh b/.github/workflows/scripts/freeze_images.sh index 2e3e0787d5..fbf175326f 100644 --- a/.github/workflows/scripts/freeze_images.sh +++ b/.github/workflows/scripts/freeze_images.sh @@ -24,9 +24,9 @@ function replace_image_version() { echo "version is empty" else echo "replace $repo_image:latest with $repo_image:$version" - find . -name "Dockerfile*" | xargs sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" - find . -name "*.yaml" | xargs sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" - find . -name "*.md" | xargs sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" + find . -name "Dockerfile*" -print0 | xargs -0 sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" + find . -name "*.yaml" -print0 | xargs -0 sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" + find . -name "*.md" -print0 | xargs -0 sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g" fi } diff --git a/.github/workflows/scripts/freeze_requirements.sh b/.github/workflows/scripts/freeze_requirements.sh index d4b6a8633a..4ff2e982cf 100644 --- a/.github/workflows/scripts/freeze_requirements.sh +++ b/.github/workflows/scripts/freeze_requirements.sh @@ -57,8 +57,8 @@ function main() { check_branch_name echo "::group::pip install --no-cache-dir pip-tools" && pip install --no-cache-dir pip-tools --upgrade && echo "::endgroup::" export -f freeze - find . -name "requirements.txt" | xargs -n 1 -I {} bash -c 'freeze "$@"' _ {} - find . -name "requirements-runtime.txt" | xargs -n 1 -I {} bash -c 'freeze "$@"' _ {} + find . -name "requirements.txt" -print0 | xargs -0 -n 1 -I {} bash -c 'freeze "$@"' _ {} + find . -name "requirements-runtime.txt" -print0 | xargs -0 -n 1 -I {} bash -c 'freeze "$@"' _ {} } main diff --git a/.github/workflows/scripts/get_test_matrix.sh b/.github/workflows/scripts/get_test_matrix.sh index 8b93662796..c810325604 100644 --- a/.github/workflows/scripts/get_test_matrix.sh +++ b/.github/workflows/scripts/get_test_matrix.sh @@ -44,7 +44,7 @@ function find_test_1() { # Accurate matching test scripts # find_test=$(find ./tests -type f \( -name test_${service_name}_${integrations}.sh -o -name test_${service_name}_${integrations}_on_*.sh \)) || true # Fuzzy matching test scripts, for example, llms/src/text-generation/integrations/opea.py match several tests. - find_test=$(find ./tests -type f -name test_${service_name}_${integration}*.sh) || true + find_test=$(find ./tests -type f -name "test_${service_name}_${integration}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" else @@ -54,7 +54,7 @@ function find_test_1() { done fi if [ "$run_all_interation" = "true" ]; then - find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true + find_test=$(find ./tests -type f -name "test_${service_name}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" fi @@ -62,7 +62,7 @@ function find_test_1() { elif [[ $(echo ${service_path} | grep "third_parties") ]]; then # new org with `src` and `third_parties` folder service_name=$(echo $service_path | sed 's:/src::' | tr '/' '_' | cut -c7-) # comps/third_parties/vllm/src -> third_parties_vllm - find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true + find_test=$(find ./tests -type f -name "test_${service_name}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" fi @@ -77,7 +77,7 @@ function find_test_1() { else # old org without 'src' folder service_name=$(echo $service_path | tr '/' '_' | cut -c7-) # comps/retrievers/redis/langchain -> retrievers_redis_langchain - find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true + find_test=$(find ./tests -type f -name "test_${service_name}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" fi @@ -140,7 +140,7 @@ function find_test_3() { _domain=${_domain#compose_} _service=${_service}_${_domain} fi - find_test=$(find ./tests -type f -name test_${_service}*.sh) || true + find_test=$(find ./tests -type f -name "test_${_service}*.sh") || true if [ "$find_test" ]; then fill_in_matrix "$find_test" fi diff --git a/.github/workflows/scripts/test_ut.sh b/.github/workflows/scripts/test_ut.sh index 5f70c9a60e..39ce87f607 100644 --- a/.github/workflows/scripts/test_ut.sh +++ b/.github/workflows/scripts/test_ut.sh @@ -10,7 +10,7 @@ cd /GenAIComps/tests if [ $test_name = 'mega' ]; then echo "run mega test" cd cores - find . -name "*.yaml" -exec sh -c 'x="{}"; cp $x ./' \; + find . -name "*.yaml" -exec sh -c 'cp "$1" ./ ' _ {} \; comps_path=$(python -c 'import comps; print(comps.__path__[0])') find . -name "test*.py" | sed "s,\.\/,python -m pytest --cov=\"${comps_path}\" --cov-report term --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run.sh bash run.sh 2>&1 | tee ${ut_log_name} diff --git a/comps/animation/src/docker_run.sh b/comps/animation/src/docker_run.sh index c6e48c257a..2384c12713 100644 --- a/comps/animation/src/docker_run.sh +++ b/comps/animation/src/docker_run.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/finetuning/src/integrations/xtune/prepare_xtune.sh b/comps/finetuning/src/integrations/xtune/prepare_xtune.sh index f35ca70d43..2fa24102ac 100644 --- a/comps/finetuning/src/integrations/xtune/prepare_xtune.sh +++ b/comps/finetuning/src/integrations/xtune/prepare_xtune.sh @@ -52,6 +52,6 @@ else ps aux | grep "llamafactory-cli webui" | head -n 1 | awk '{print $2}' |xargs kill echo "kill webui service done" fi - echo 0 >> done + echo 0 >> "done" echo "Please follow README.md to install driver or update torch lib" fi diff --git a/comps/finetuning/src/launch.sh b/comps/finetuning/src/launch.sh index d2438bf646..b0eabe10d4 100644 --- a/comps/finetuning/src/launch.sh +++ b/comps/finetuning/src/launch.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/guardrails/src/hallucination_detection/build_docker_hallucination_microservice.sh b/comps/guardrails/src/hallucination_detection/build_docker_hallucination_microservice.sh index 3e4da24ecb..d11af098a5 100644 --- a/comps/guardrails/src/hallucination_detection/build_docker_hallucination_microservice.sh +++ b/comps/guardrails/src/hallucination_detection/build_docker_hallucination_microservice.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/guardrails/src/hallucination_detection/launch_hallucination_microservice.sh b/comps/guardrails/src/hallucination_detection/launch_hallucination_microservice.sh index db52fd4c3e..c5d4143119 100644 --- a/comps/guardrails/src/hallucination_detection/launch_hallucination_microservice.sh +++ b/comps/guardrails/src/hallucination_detection/launch_hallucination_microservice.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/struct2graph/src/environment_setup.sh b/comps/struct2graph/src/environment_setup.sh index 5bb0823b35..215015cefa 100644 --- a/comps/struct2graph/src/environment_setup.sh +++ b/comps/struct2graph/src/environment_setup.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/text2graph/src/setup_service_env.sh b/comps/text2graph/src/setup_service_env.sh index be61581162..518902abf6 100755 --- a/comps/text2graph/src/setup_service_env.sh +++ b/comps/text2graph/src/setup_service_env.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/comps/text2kg/src/environment_setup.sh b/comps/text2kg/src/environment_setup.sh index 3a25a771ad..7b1637629d 100644 --- a/comps/text2kg/src/environment_setup.sh +++ b/comps/text2kg/src/environment_setup.sh @@ -28,8 +28,8 @@ export NEO4J_USERNAME=${NEO4J_USERNAME:-"neo4j"} export NEO4J_PASSWORD=${NEO4J_PASSWORD:-"neo4j_password"} export NEO4J_URL=${NEO4J_URL:-"neo4j://localhost:7687"} export NEO4J_URI=${NEO4J_URI:-"neo4j://localhost:7687"} -export NEO4J_PORT1={$NEO4J_PORT1:-7474}:7474 -export NEO4J_PORT2={$NEO4J_PORT2:-7687}:7687 +export NEO4J_PORT1=${NEO4J_PORT1:-7474}:7474 +export NEO4J_PORT2=${NEO4J_PORT2:-7687}:7687 export NEO4J_AUTH=neo4j/password export NEO4J_PLUGINS=\[\"apoc\"\] export NEO4J_HEALTH_URL="http://localhost:7474" diff --git a/comps/third_parties/video-llama/src/start.sh b/comps/third_parties/video-llama/src/start.sh index 555102f805..3b31512d14 100644 --- a/comps/third_parties/video-llama/src/start.sh +++ b/comps/third_parties/video-llama/src/start.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # Download models diff --git a/comps/third_parties/wav2lip/src/download_ckpts.sh b/comps/third_parties/wav2lip/src/download_ckpts.sh index ae28fc23b4..fd4ce465f1 100644 --- a/comps/third_parties/wav2lip/src/download_ckpts.sh +++ b/comps/third_parties/wav2lip/src/download_ckpts.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/agent/build_vllm_gaudi.sh b/tests/agent/build_vllm_gaudi.sh index 6c2aaf39cc..90bb70cebe 100644 --- a/tests/agent/build_vllm_gaudi.sh +++ b/tests/agent/build_vllm_gaudi.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/agent/launch_vllm_gaudi.sh b/tests/agent/launch_vllm_gaudi.sh index 1e77969bb8..0eca77cd19 100644 --- a/tests/agent/launch_vllm_gaudi.sh +++ b/tests/agent/launch_vllm_gaudi.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/agent/sql_agent_test/run_data_split.sh b/tests/agent/sql_agent_test/run_data_split.sh index 72ca895d8e..07f1d71474 100644 --- a/tests/agent/sql_agent_test/run_data_split.sh +++ b/tests/agent/sql_agent_test/run_data_split.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/agent/test_agent_langchain_on_intel_hpu.sh b/tests/agent/test_agent_langchain_on_intel_hpu.sh index ceef5ee27b..da27833b4e 100644 --- a/tests/agent/test_agent_langchain_on_intel_hpu.sh +++ b/tests/agent/test_agent_langchain_on_intel_hpu.sh @@ -453,7 +453,7 @@ function main() { stop_docker - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } main diff --git a/tests/guardrails/test_guardrails_bias_detection_distilbias.sh b/tests/guardrails/test_guardrails_bias_detection_distilbias.sh index e47ecce238..90cb2f574f 100644 --- a/tests/guardrails/test_guardrails_bias_detection_distilbias.sh +++ b/tests/guardrails/test_guardrails_bias_detection_distilbias.sh @@ -70,7 +70,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh b/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh index da77898a13..bdf65346ca 100644 --- a/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh +++ b/tests/guardrails/test_guardrails_guardrails_llamaguard_on_intel_hpu.sh @@ -41,7 +41,7 @@ function start_service() { function validate_microservice() { echo "Validate microservice started" echo "test 1 - violated policies" - result=$(http_proxy= curl http://localhost:11303/v1/guardrails -X POST -d '{"text":"How do you buy a tiger in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') + result=$(http_proxy="" curl http://localhost:11303/v1/guardrails -X POST -d '{"text":"How do you buy a tiger in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') if [[ $result == *"Violated"* ]]; then echo "Result correct." else @@ -51,7 +51,7 @@ function validate_microservice() { exit 1 fi echo "test 2 - safe" - result=$(http_proxy= curl http://localhost:11303/v1/guardrails -X POST -d '{"text":"How do you buy a car in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') + result=$(http_proxy="" curl http://localhost:11303/v1/guardrails -X POST -d '{"text":"How do you buy a car in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') if [[ $result == *"car"* ]]; then echo "Result correct." else @@ -80,7 +80,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh b/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh index 054ac861a7..34a674bfaa 100644 --- a/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh +++ b/tests/guardrails/test_guardrails_guardrails_wildguard_on_intel_hpu.sh @@ -41,7 +41,7 @@ function start_service() { function validate_microservice() { echo "Validate microservice started" echo "test 1 - violated policies" - result=$(http_proxy= curl http://localhost:11304/v1/guardrails -X POST -d '{"text":"How do you buy a tiger in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') + result=$(http_proxy="" curl http://localhost:11304/v1/guardrails -X POST -d '{"text":"How do you buy a tiger in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') if [[ $result == *"Violated"* ]]; then echo "Result correct." else @@ -51,7 +51,7 @@ function validate_microservice() { exit 1 fi echo "test 2 - safe" - result=$(http_proxy= curl http://localhost:11304/v1/guardrails -X POST -d '{"text":"How do you buy a car in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') + result=$(http_proxy="" curl http://localhost:11304/v1/guardrails -X POST -d '{"text":"How do you buy a car in the US?","parameters":{"max_new_tokens":32}}' -H 'Content-Type: application/json') if [[ $result == *"car"* ]]; then echo "Result correct." else @@ -79,7 +79,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_hallucination_detection_on_intel_hpu.sh b/tests/guardrails/test_guardrails_hallucination_detection_on_intel_hpu.sh index 34abccf7a1..eb99fddc34 100644 --- a/tests/guardrails/test_guardrails_hallucination_detection_on_intel_hpu.sh +++ b/tests/guardrails/test_guardrails_hallucination_detection_on_intel_hpu.sh @@ -115,7 +115,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_prompt_injection_promptguard.sh b/tests/guardrails/test_guardrails_prompt_injection_promptguard.sh index c6bd20038a..03e84c128f 100644 --- a/tests/guardrails/test_guardrails_prompt_injection_promptguard.sh +++ b/tests/guardrails/test_guardrails_prompt_injection_promptguard.sh @@ -112,7 +112,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/guardrails/test_guardrails_toxicity_detection_toxicdetection.sh b/tests/guardrails/test_guardrails_toxicity_detection_toxicdetection.sh index cb172bbd75..905b27847c 100644 --- a/tests/guardrails/test_guardrails_toxicity_detection_toxicdetection.sh +++ b/tests/guardrails/test_guardrails_toxicity_detection_toxicdetection.sh @@ -66,7 +66,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/llms/llm_utils.sh b/tests/llms/llm_utils.sh index b86f8283aa..d06470d6be 100644 --- a/tests/llms/llm_utils.sh +++ b/tests/llms/llm_utils.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/third_parties/_test_third_parties_sglang.sh b/tests/third_parties/_test_third_parties_sglang.sh index 4a97c88ac7..170f7488f2 100644 --- a/tests/third_parties/_test_third_parties_sglang.sh +++ b/tests/third_parties/_test_third_parties_sglang.sh @@ -77,7 +77,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/third_parties/test_third_parties_ipex.sh b/tests/third_parties/test_third_parties_ipex.sh index e172148c33..b5dd9cfa8d 100644 --- a/tests/third_parties/test_third_parties_ipex.sh +++ b/tests/third_parties/test_third_parties_ipex.sh @@ -71,7 +71,7 @@ function main() { stop_docker echo "cleanup container images and volumes" - echo y | docker system prune 2>&1 > /dev/null + echo y | docker system prune > /dev/null 2>&1 } diff --git a/tests/utils/wait-for-it.sh b/tests/utils/wait-for-it.sh index 2c77965bc8..bbadc648ae 100755 --- a/tests/utils/wait-for-it.sh +++ b/tests/utils/wait-for-it.sh @@ -61,7 +61,7 @@ wait_for_wrapper() timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & fi WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT + trap 'kill -INT -$WAITFORIT_PID' INT wait $WAITFORIT_PID WAITFORIT_RESULT=$? if [[ $WAITFORIT_RESULT -ne 0 ]]; then