code cleaninig

Signed-off-by: Mustafa <[email protected]>

Author: MSCetin37

tests/retrievers/test_retrievers_milvus.sh

@@ -13,8 +13,7 @@ echo "TAG=${TAG}"
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 export host_ip=$(hostname -I | awk '{print $1}')
-service_name="retriever-milvus etcd minio standalone"
-retriever_service_name="retriever-milvus"
+service_name="retriever-milvus"
 
 function build_docker_images() {
     cd $WORKPATH
@@ -28,7 +27,7 @@ function build_docker_images() {
 }
 
 function start_service() {
-    export MINIO_PORT1=11611
+    export MINIO_PORT1=11611   # 11600-11699
     export MINIO_PORT2=11612
     export MILVUS_STANDALONE_PORT=11613
     export TEI_EMBEDDER_PORT=11614
@@ -39,6 +38,13 @@ function start_service() {
     export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}"
     export MILVUS_HOST=${host_ip}
 
+    # wget https://raw.githubusercontent.com/milvus-io/milvus/v2.4.9/configs/milvus.yaml
+    # wget https://github.com/milvus-io/milvus/releases/download/v2.4.9/milvus-standalone-docker-compose.yml -O docker-compose.yml
+    # sed '/- \${DOCKER_VOLUME_DIRECTORY:-\.}\/volumes\/milvus:\/var\/lib\/milvus/a \ \ \ \ \ \ - \${DOCKER_VOLUME_DIRECTORY:-\.}\/milvus.yaml:\/milvus\/configs\/milvus.yaml' -i docker-compose.yml
+
+    cd $WORKPATH/comps/third_parties/milvus/deployment/docker_compose/
+    docker compose -f compose.yaml up -d > ${LOG_PATH}/start_services_with_compose_milvus.log
+
     cd $WORKPATH/comps/retrievers/deployment/docker_compose
     docker compose -f compose.yaml up ${service_name} -d > ${LOG_PATH}/start_services_with_compose.log
 
@@ -61,12 +67,12 @@ function validate_microservice() {
             echo "[ retriever ] Content is as expected."
         else
             echo "[ retriever ] Content does not match the expected result: $CONTENT"
-            docker logs ${retriever_service_name} >> ${LOG_PATH}/retriever.log
+            docker logs ${service_name} >> ${LOG_PATH}/retriever.log
             exit 1
         fi
     else
         echo "[ retriever ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${retriever_service_name} >> ${LOG_PATH}/retriever.log
+        docker logs ${service_name} >> ${LOG_PATH}/retriever.log
         exit 1
     fi
 }

tests/retrievers/test_retrievers_neo4j_on_intel_hpu.sh

@@ -39,6 +39,8 @@ function build_docker_images() {
 }
 
 function start_service() {
+    export NEO4J_PORT1=11631
+    export NEO4J_PORT2=11632
     export TEI_EMBEDDER_PORT=11633
     export LLM_ENDPOINT_PORT=11634
     export RETRIEVER_PORT=11635
@@ -50,10 +52,7 @@ function start_service() {
     export LLM_MODEL_ID="meta-llama/Meta-Llama-3.1-8B-Instruct"
     export TGI_LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}"
     export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6004"
-    export NEO4J_PORT1=7474   # 11631
-    export NEO4J_PORT2=7687   # 11632
     export NEO4J_URI="bolt://${host_ip}:${NEO4J_PORT2}"
-    export NEO4J_URL="bolt://${host_ip}:${NEO4J_PORT2}"
     export NEO4J_USERNAME="neo4j"
     export NEO4J_PASSWORD="neo4jtest"
     export no_proxy="localhost,127.0.0.1,"${host_ip}
@@ -66,10 +65,9 @@ function start_service() {
     # dataprep neo4j
     # Not testing openai code path since not able to provide key for cicd
     docker run -d --name="test-comps-retrievers-neo4j-llama-index-dataprep" -p 6004:5000 -v ./data:/data --ipc=host -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT \
-        -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT -e EMBEDDING_MODEL_ID=$EMBEDDING_MODEL_ID -e LLM_MODEL_ID=$LLM_MODEL_ID -e host_ip=$host_ip -e no_proxy=$no_proxy \
+        -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT -e TEI_ENDPOINT=$TEI_EMBEDDING_ENDPOINT -e EMBEDDING_MODEL_ID=$EMBEDDING_MODEL_ID -e LLM_MODEL_ID=$LLM_MODEL_ID -e host_ip=$host_ip -e no_proxy=$no_proxy \
         -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e NEO4J_URL="bolt://${host_ip}:${NEO4J_PORT2}" -e NEO4J_USERNAME="neo4j" \
-        -e NEO4J_PASSWORD="neo4jtest" -e NEO4J_PORT1=$NEO4J_PORT1 -e NEO4J_PORT2=$NEO4J_PORT2 -e HF_TOKEN=$HF_TOKEN -e MAX_INPUT_TOKENS=$MAX_INPUT_TOKENS -e LOGFLAG=True \
-        -e DATAPREP_COMPONENT_NAME="OPEA_DATAPREP_NEO4J_LLAMAINDEX" opea/dataprep-neo4j-llamaindex:comps
+        -e NEO4J_PASSWORD="neo4jtest" -e HF_TOKEN=$HF_TOKEN -e MAX_INPUT_LEN=$MAX_INPUT_TOKENS -e LOGFLAG=True -e DATAPREP_COMPONENT_NAME="OPEA_DATAPREP_NEO4J_LLAMAINDEX" opea/dataprep-neo4j-llamaindex:comps
 
     sleep 1m
 

tests/retrievers/test_retrievers_opensearch.sh

@@ -14,7 +14,6 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 export host_ip=$(hostname -I | awk '{print $1}')
 service_name="opensearch-vector-db tei-embedding-serving retriever-opensearch"
-retriever_service_name="retriever-opensearch"
 
 function build_docker_images() {
     cd $WORKPATH
@@ -35,7 +34,6 @@ function start_service() {
     export OPENSEARCH_INITIAL_ADMIN_PASSWORD="StRoNgOpEa0)"
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}"
-    export HUGGINGFACEHUB_API_TOKEN=${HF_TOKEN}
     export OPENSEARCH_URL="http://${host_ip}:${OPENSEARCH_PORT1}"
     export INDEX_NAME="file-index"
 
@@ -62,13 +60,13 @@ function validate_microservice() {
             echo "[ retriever ] Content is as expected."
         else
             echo "[ retriever ] Content does not match the expected result: $CONTENT"
-            docker logs ${retriever_service_name} >> ${LOG_PATH}/retriever.log
+            docker logs ${service_name} >> ${LOG_PATH}/retriever.log
             docker logs tei-embedding-serving >> ${LOG_PATH}/tei.log
             exit 1
         fi
     else
         echo "[ retriever ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${retriever_service_name} >> ${LOG_PATH}/retriever.log
+        docker logs ${service_name} >> ${LOG_PATH}/retriever.log
         docker logs tei-embedding-serving >> ${LOG_PATH}/tei.log
         exit 1
     fi
@@ -82,14 +80,14 @@ function stop_docker() {
 }
 
 function main() {
-    stop_docker
+    stop_service
 
     build_docker_images
     start_service
 
     validate_microservice
 
-    stop_docker
+    stop_service
     echo y | docker system prune
 }
 

tests/retrievers/test_retrievers_vdms.sh

@@ -37,7 +37,7 @@ function start_service() {
     export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}"
     export INDEX_NAME="rag-vdms"
     export VDMS_USE_CLIP=0 #set to 1 if openai clip embedding should be used
-    export HUGGINGFACEHUB_API_TOKEN=${HF_TOKEN}
+    export HF_TOKEN=${HF_TOKEN}
     export LOGFLAG=True
 
     cd $WORKPATH/comps/retrievers/deployment/docker_compose