diff --git a/.github/workflows/rocm.yml b/.github/workflows/rocm.yml
index 86b6412ad4fc..725aa9f14757 100644
--- a/.github/workflows/rocm.yml
+++ b/.github/workflows/rocm.yml
@@ -23,7 +23,15 @@ jobs:
         docker build -t rocm -f Dockerfile.rocm .
     - name: Run Sanity Test
       run: |
-        docker run --detached --p 8000:8000 rocm
+        # detele any existing container
+        docker rm -f $(docker ps -a -q)
+        # run the container
+        docker run -d --p 8000:8000 rocm
+        # wait for the server to start
         while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000)" != "200" ]; do sleep 1; done
+        # run the sanity test
         python examples/openai_completion_client.py
+        # remove the container
+        docker rm -f $(docker ps -a -q)
+