feat(vllm-model): on-policy sampling pin via sampling_overrides

Add a framework-agnostic sampling pin to vllm_model: sampling_overrides forces
temperature/top_p on every chat request, read from generic policy_generation_*
keys with on-policy defaults. This keeps an external harness's rollouts
on-policy for training without Gym holding any framework-specific knowledge —
the training framework publishes its sampling into the generic keys.

Signed-off-by: Ananth Subramaniam <ansubramania@nvidia.com>

Author: ananthsub

responses_api_models/vllm_model/app.py

@@ -66,6 +66,13 @@ class VLLMModelConfig(BaseResponsesAPIModelConfig):
     # Corresponds to the extra_body of OpenAI Client.
     extra_body: Optional[Dict[str, Any]] = None
 
+    # Sampling params to force on every chat request, overriding whatever the client sent. An
+    # external harness (e.g. a CLI agent) chooses its own temperature/top_p, but on-policy RL
+    # training requires generation to match the sampling distribution the policy is optimized under.
+    # The integrating training framework sets these to its generation sampling params; Gym only
+    # enforces them and holds no knowledge of any specific framework. Unset means no override.
+    sampling_overrides: Optional[Dict[str, Any]] = None
+
     default_headers: Dict[str, str] = Field(default_factory=dict)
     # Optional prefix for resolving relative ``metadata.audio_path`` (or
     # entries in ``metadata.audio_paths``) against. Absolute paths are used
@@ -379,6 +386,12 @@ def _preprocess_chat_completion_create_params(self, request: Request, body_dict:
                 # No user message found — create one with just the audio blocks.
                 body_dict.setdefault("messages", []).append({"role": "user", "content": list(audio_blocks)})
 
+        # Pin sampling params last so they win over anything the client sent. On-policy RL training
+        # requires the generation to match the training worker's sampling config; an external harness
+        # sets its own temperature/top_p, so force them here to keep captured rollouts on-policy.
+        if self.config.sampling_overrides:
+            body_dict.update(self.config.sampling_overrides)
+
         return body_dict
 
     async def chat_completions(

responses_api_models/vllm_model/configs/vllm_model_for_training.yaml

@@ -7,3 +7,10 @@ policy_model:
       model: ${policy_model_name}
       return_token_id_information: true
       uses_reasoning_parser: true
+      # On-policy training: force generation to the sampling params the policy is optimized under,
+      # overriding whatever an external harness requests, so captured rollouts stay on-policy. The
+      # integrating training framework supplies these via the generic keys below (Gym does not know
+      # any framework); the defaults are standard on-policy sampling when a framework sets nothing.
+      sampling_overrides:
+        temperature: ${oc.select:policy_generation_temperature,1.0}
+        top_p: ${oc.select:policy_generation_top_p,1.0}