From ce697749bb14101c73430e971807f219c1655a27 Mon Sep 17 00:00:00 2001 From: tunji-ruwase_snow Date: Mon, 29 Jun 2026 16:36:53 +0000 Subject: [PATCH 1/2] Fix vllm port toctou --- arctic_inference/server/replica_pool.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arctic_inference/server/replica_pool.py b/arctic_inference/server/replica_pool.py index 903d253ef..c1e35b408 100644 --- a/arctic_inference/server/replica_pool.py +++ b/arctic_inference/server/replica_pool.py @@ -265,6 +265,24 @@ async def initialize( rid = self._bundle_indices[i] env["VLLM_RAY_BUNDLE_INDICES"] = ",".join( str(rid * self.tp_size + t) for t in range(self.tp_size)) + # Pin a deterministic, per-replica base port. When many colocated + # TP engines initialize concurrently on the same node, vLLM's + # get_open_port() has a TOCTOU window: it binds a probe socket, + # reads the port, closes it, and returns the number, so several + # engines can receive the same port before any binds the TP + # rendezvous TCPStore (tcp://127.0.0.1:) -> EADDRINUSE. + # Giving each replica a distinct VLLM_PORT base makes get_open_port + # start from a unique value per engine; vLLM still increments from + # the base on any residual conflict. The globally-unique replica + # index i guarantees uniqueness across replicas colocated on the + # same node. Disable via ARCTIC_DISABLE_VLLM_PORT_ASSIGN=1; tune the + # range via ARCTIC_VLLM_PORT_BASE / ARCTIC_VLLM_PORT_STRIDE. + if (self.tp_size > 1 + and os.environ.get("ARCTIC_DISABLE_VLLM_PORT_ASSIGN") != "1"): + stride = int(os.environ.get("ARCTIC_VLLM_PORT_STRIDE", "100")) + base = int(env.get("VLLM_PORT", + os.environ.get("ARCTIC_VLLM_PORT_BASE", "30000"))) + env["VLLM_PORT"] = str(base + i * stride) init_tasks.append(w.initialize.remote(engine_kwargs, env or None)) await asyncio.gather(*init_tasks) From d2830d72974f5ecff6507dec99ceb4ba615bdb8b Mon Sep 17 00:00:00 2001 From: tunji-ruwase_snow Date: Mon, 29 Jun 2026 17:27:26 +0000 Subject: [PATCH 2/2] Simplify per Mike's suggestion --- arctic_inference/server/replica_pool.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/arctic_inference/server/replica_pool.py b/arctic_inference/server/replica_pool.py index c1e35b408..adf511c1b 100644 --- a/arctic_inference/server/replica_pool.py +++ b/arctic_inference/server/replica_pool.py @@ -265,24 +265,12 @@ async def initialize( rid = self._bundle_indices[i] env["VLLM_RAY_BUNDLE_INDICES"] = ",".join( str(rid * self.tp_size + t) for t in range(self.tp_size)) - # Pin a deterministic, per-replica base port. When many colocated - # TP engines initialize concurrently on the same node, vLLM's - # get_open_port() has a TOCTOU window: it binds a probe socket, - # reads the port, closes it, and returns the number, so several - # engines can receive the same port before any binds the TP - # rendezvous TCPStore (tcp://127.0.0.1:) -> EADDRINUSE. - # Giving each replica a distinct VLLM_PORT base makes get_open_port - # start from a unique value per engine; vLLM still increments from - # the base on any residual conflict. The globally-unique replica - # index i guarantees uniqueness across replicas colocated on the - # same node. Disable via ARCTIC_DISABLE_VLLM_PORT_ASSIGN=1; tune the - # range via ARCTIC_VLLM_PORT_BASE / ARCTIC_VLLM_PORT_STRIDE. - if (self.tp_size > 1 - and os.environ.get("ARCTIC_DISABLE_VLLM_PORT_ASSIGN") != "1"): - stride = int(os.environ.get("ARCTIC_VLLM_PORT_STRIDE", "100")) - base = int(env.get("VLLM_PORT", - os.environ.get("ARCTIC_VLLM_PORT_BASE", "30000"))) - env["VLLM_PORT"] = str(base + i * stride) + # Ensure we start from offset base ports so that vllm workers do not + # collide when attempting to obtain free ports. vLLM increments ports + # until a free one is found, but this can lead to a race condition when + # spinning up many replicas at the same time. + base, stride = 30000, 100 + env["VLLM_PORT"] = str(base + i*stride) init_tasks.append(w.initialize.remote(engine_kwargs, env or None)) await asyncio.gather(*init_tasks)