diff --git a/comps/embeddings/deployment/docker_compose/compose.yaml b/comps/embeddings/deployment/docker_compose/compose.yaml index c184af6d7e..9b7c7daaf2 100644 --- a/comps/embeddings/deployment/docker_compose/compose.yaml +++ b/comps/embeddings/deployment/docker_compose/compose.yaml @@ -101,6 +101,20 @@ services: multimodal-bridgetower-embedding-gaudi-serving: condition: service_healthy + multimodal-bridgetower-embedding-server-offline: + <<: *multimodal-bridgetower-embedding-config + container_name: embedding-multimodal-bridgetower-server-offline + depends_on: + multimodal-bridgetower-embedding-serving-offline: + condition: service_healthy + + multimodal-bridgetower-embedding-gaudi-server-offline: + <<: *multimodal-bridgetower-embedding-config + container_name: embedding-multimodal-bridgetower-gaudi-server-offline + depends_on: + multimodal-bridgetower-embedding-gaudi-serving-offline: + condition: service_healthy + networks: default: driver: bridge diff --git a/comps/embeddings/src/README_bridgetower.md b/comps/embeddings/src/README_bridgetower.md index 4739283690..87acf3bf5f 100644 --- a/comps/embeddings/src/README_bridgetower.md +++ b/comps/embeddings/src/README_bridgetower.md @@ -105,3 +105,34 @@ curl -X POST http://0.0.0.0:6600/v1/embeddings \ ``` This request will return an embedding representing the semantic meaning of the input text. + +## Running in the air gapped environment + +The following steps are common for running the dataprep microservice in an air gapped environment (a.k.a. environment with no internet access). + +1. Download the necessary data + +```shell +# Download model +export DATA_PATH="" +huggingface-cli download --cache-dir $DATA_PATH BridgeTower/bridgetower-large-itm-mlm-itc +# Download image for warmup +cd $DATA_PATH +wget https://llava-vl.github.io/static/images/view.jpg +``` + +2. launch the `embedding-multimodal-bridgetower` microservice with the following settings: + +- For Gaudi HPU: + +```bash +export DATA_PATH="" +docker compose up multimodal-bridgetower-embedding-gaudi-serving-offline multimodal-bridgetower-embedding-gaudi-server-offline -d +``` + +- For Xeon CPU: + +```bash +export DATA_PATH="" +docker compose up multimodal-bridgetower-embedding-serving-offline multimodal-bridgetower-embedding-server-offline -d +``` diff --git a/comps/third_parties/bridgetower/deployment/docker_compose/compose.yaml b/comps/third_parties/bridgetower/deployment/docker_compose/compose.yaml index 161439256e..450e52c5c2 100644 --- a/comps/third_parties/bridgetower/deployment/docker_compose/compose.yaml +++ b/comps/third_parties/bridgetower/deployment/docker_compose/compose.yaml @@ -45,6 +45,59 @@ services: retries: 18 start_period: 30s + multimodal-bridgetower-embedding-serving-offline: + image: ${REGISTRY:-opea}/embedding-multimodal-bridgetower:${TAG:-latest} + container_name: multimodal-bridgetower-embedding-serving-offline + ports: + - ${EMBEDDER_PORT:-12400}:${EMBEDDER_PORT:-12400} + ipc: host + environment: + HF_HUB_OFFLINE: 1 + HF_HUB_CACHE: /data + # Use non-existing proxy to mimic air gapped environment + no_proxy: localhost,127.0.0.1,${offline_no_proxy} + http_proxy: http://localhost:7777 + https_proxy: http://localhost:7777 + PORT: ${EMBEDDER_PORT} + restart: unless-stopped + volumes: + - "${DATA_PATH:-./data}:/data" + healthcheck: + test: ["CMD-SHELL", "http_proxy='' curl -f http://localhost:${EMBEDDER_PORT}/v1/health_check"] + interval: 10s + timeout: 6s + retries: 18 + start_period: 30s + + + multimodal-bridgetower-embedding-gaudi-serving-offline: + image: ${REGISTRY:-opea}/embedding-multimodal-bridgetower-gaudi:${TAG:-latest} + container_name: multimodal-bridgetower-embedding-gaudi-serving-offline + ports: + - ${EMBEDDER_PORT:-12400}:${EMBEDDER_PORT:-12400} + ipc: host + environment: + HF_HUB_OFFLINE: 1 + HF_HUB_CACHE: /data + # Use non-existing proxy to mimic air gapped environment + no_proxy: localhost,127.0.0.1,${offline_no_proxy} + http_proxy: http://localhost:7777 + https_proxy: http://localhost:7777 + PORT: ${EMBEDDER_PORT} + HABANA_VISIBLE_DEVICES: all + volumes: + - "${DATA_PATH:-./data}:/data" + runtime: habana + cap_add: + - SYS_NICE + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "http_proxy='' curl -f http://localhost:${EMBEDDER_PORT}/v1/health_check"] + interval: 10s + timeout: 6s + retries: 18 + start_period: 30s + networks: default: driver: bridge diff --git a/comps/third_parties/bridgetower/src/README.md b/comps/third_parties/bridgetower/src/README.md index 1d87a7b9c2..bd78efd70b 100644 --- a/comps/third_parties/bridgetower/src/README.md +++ b/comps/third_parties/bridgetower/src/README.md @@ -44,6 +44,39 @@ cd comps/third_parties/bridgetower/deployment/docker_compose/ docker compose -f compose.yaml up -d multimodal-bridgetower-embedding-serving ``` +## Running in the air gapped environment + +The following steps are common for running the dataprep microservice in an air gapped environment (a.k.a. environment with no internet access). + +1. Download the necessary data + +```shell +# Download model +export DATA_PATH="" +huggingface-cli download --cache-dir $DATA_PATH BridgeTower/bridgetower-large-itm-mlm-itc +# Download image for warmup +cd $DATA_PATH +wget https://llava-vl.github.io/static/images/view.jpg +``` + +2. launch the `embedding-multimodal-bridgetower` microservice with the following settings: + +- For Gaudi HPU: + +```bash +export DATA_PATH="" +cd comps/third_parties/bridgetower/deployment/docker_compose/ +docker compose -f compose.yaml up -d multimodal-bridgetower-embedding-gaudi-serving-offline +``` + +- For Xeon CPU: + +```bash +export DATA_PATH="" +cd comps/third_parties/bridgetower/deployment/docker_compose/ +docker compose -f compose.yaml up -d multimodal-bridgetower-embedding-serving-offline +``` + ## 🚀3. Access the service Then you need to test your MMEI service using the following commands: diff --git a/comps/third_parties/bridgetower/src/bridgetower_server.py b/comps/third_parties/bridgetower/src/bridgetower_server.py index f0a06897b0..50e5236060 100644 --- a/comps/third_parties/bridgetower/src/bridgetower_server.py +++ b/comps/third_parties/bridgetower/src/bridgetower_server.py @@ -131,11 +131,18 @@ async def get_woker_status(): # warmup print("Warmup...") - image_paths = ["https://llava-vl.github.io/static/images/view.jpg"] + is_offline = os.getenv("HF_HUB_OFFLINE") == "1" + if is_offline: + image_paths = ["/data/view.jpg"] + else: + image_paths = ["https://llava-vl.github.io/static/images/view.jpg"] example_prompts = ["This is test image!"] images = [] for image_path in image_paths: - images.append(PIL.Image.open(requests.get(image_path, stream=True, timeout=3000).raw)) + if is_offline: + images.append(PIL.Image.open(image_path)) + else: + images.append(PIL.Image.open(requests.get(image_path, stream=True, timeout=3000).raw)) for i in range(args.warmup): embedder.embed_image_text_pairs( example_prompts,