diff --git a/python/huggingface_server.Dockerfile b/python/huggingface_server.Dockerfile
index 37f3fc32b85..7f3947a213f 100644
--- a/python/huggingface_server.Dockerfile
+++ b/python/huggingface_server.Dockerfile
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=nvidia/cuda:12.1.0-devel-ubuntu22.04
+ARG BASE_IMAGE=nvidia/cuda:12.6.3-devel-ubuntu22.04
 ARG VENV_PATH=/prod_venv
 
 FROM ${BASE_IMAGE} as builder
@@ -33,7 +33,7 @@ RUN cd huggingfaceserver && poetry install --no-interaction --no-cache
 
 RUN pip3 install vllm==${VLLM_VERSION}
 
-FROM nvidia/cuda:12.1.0-base-ubuntu22.04 as prod
+FROM nvidia/cuda:12.6.3-base-ubuntu22.04 as prod
 
 RUN apt-get update -y && apt-get install python3.10-venv -y