From 3cf556a6b8c983db5cc361d3c0586d6b4336fd17 Mon Sep 17 00:00:00 2001 From: Max Dubrinsky Date: Thu, 30 Jul 2026 13:37:47 -0400 Subject: [PATCH 1/4] feat(models): add typed NemoClient models foundation Introduces the typed Models service client that the AIRCORE-876 consumer migration will build on: request/response DTOs (types), PreparedRequest endpoint builders (endpoints), and the sync/async ModelsClient surface (client). Purely additive: no existing code imports it yet, so it changes no runtime behavior and carries zero risk to current consumers. Also carries the method() descriptor fix that this client requires -- class-level attribute access now resolves without invoking the wrapped callable, so Mock(spec=ModelsClient) and other introspection no longer break -- plus a response docstring note on distinguishing 202/204 deletes. The consumer repoint, the packages/models resources rewrite, and the vendored SDK sync land separately as the breaking, coupled steps. Covered by endpoint-builder, client-surface, and descriptor tests. Signed-off-by: Max Dubrinsky --- .../src/nemo_platform_plugin/client/method.py | 38 +- .../nemo_platform_plugin/client/response.py | 4 + .../src/nemo_platform_plugin/models/client.py | 420 +++++ .../nemo_platform_plugin/models/endpoints.py | 390 +++++ .../src/nemo_platform_plugin/models/types.py | 1466 +++++++++++++++++ .../tests/client/test_method.py | 150 ++ .../tests/models/test_client.py | 405 +++++ .../tests/models/test_endpoints.py | 312 ++++ 8 files changed, 3184 insertions(+), 1 deletion(-) create mode 100644 packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py create mode 100644 packages/nemo_platform_plugin/src/nemo_platform_plugin/models/endpoints.py create mode 100644 packages/nemo_platform_plugin/src/nemo_platform_plugin/models/types.py create mode 100644 packages/nemo_platform_plugin/tests/client/test_method.py create mode 100644 packages/nemo_platform_plugin/tests/models/test_client.py create mode 100644 packages/nemo_platform_plugin/tests/models/test_endpoints.py diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/method.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/method.py index 58c9184feb..71e14ef12b 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/method.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/method.py @@ -71,16 +71,52 @@ class EndpointMethod(Generic[P, SyncReturnT, AsyncReturnT]): the response type that ``send()`` returns for each endpoint marker. """ + # Copied from the endpoint in __init__ so help() and autodoc describe the + # endpoint. Declared here so the descriptor's introspection surface is part of + # its type rather than something callers have to discover at runtime. + __wrapped__: Callable[P, PreparedRequest] + __name__: str + __qualname__: str + __doc__: str | None + __module__: str + def __init__(self, endpoint_fn: Callable[P, PreparedRequest]) -> None: self._endpoint_fn = endpoint_fn + # Carry the endpoint's name, docstring, and annotations onto the descriptor + # so help() and autodoc describe the endpoint rather than the descriptor. + # Set directly rather than via functools.update_wrapper, which expects a + # callable wrapper; a descriptor is not one, and which would also copy + # __dict__ and with it the endpoint's __isabstractmethod__ marker. + # + # This does NOT make inspect.signature(SomeClient.method) work: signature() + # rejects a non-callable before it ever consults __wrapped__. Reach the + # parameter list via inspect.unwrap() at class level, or just read it off + # an instance, where __get__ hands back the bound function. + self.__wrapped__ = endpoint_fn + for attr in functools.WRAPPER_ASSIGNMENTS: + try: + setattr(self, attr, getattr(endpoint_fn, attr)) + except AttributeError: + pass + + @property + def endpoint(self) -> Callable[P, PreparedRequest]: + """The endpoint function this descriptor binds.""" + return self._endpoint_fn + @overload + def __get__(self, obj: None, objtype: type | None = None) -> EndpointMethod[P, SyncReturnT, AsyncReturnT]: ... @overload def __get__(self, obj: NemoClient, objtype: type | None = None) -> Callable[P, SyncReturnT]: ... @overload def __get__(self, obj: AsyncNemoClient, objtype: type | None = None) -> Callable[P, Awaitable[AsyncReturnT]]: ... def __get__(self, obj: NemoClient | AsyncNemoClient | None, objtype: type | None = None) -> object: - assert obj is not None + if obj is None: + # Class-level access. Anything that inspects a client class rather than + # an instance -- Mock(spec=...), inspect, help(), autodoc -- lands here, + # and the descriptor protocol says to hand back the descriptor itself. + return self if isinstance(obj, AsyncNemoClient): @functools.wraps(self._endpoint_fn) diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/response.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/response.py index 56c6959220..627eef150e 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/response.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/response.py @@ -79,6 +79,10 @@ class NemoResponse(Generic[ResponseT]): resp.http_response # full httpx.Response user = resp.data() # raises on non-2xx, otherwise returns body + + When several 2xx codes share one typed body (e.g. a delete that returns 202 + Accepted for async teardown or 204 No Content when already gone, both typed + ``None``), inspect ``resp.http_response.status_code`` to tell them apart. """ http_response: httpx.Response diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py new file mode 100644 index 0000000000..914f50c43b --- /dev/null +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py @@ -0,0 +1,420 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Typed HTTP clients for the Models service. + +Wraps the endpoint functions from ``models.endpoints`` as direct methods using +the ``method()`` descriptor (the Files/Secrets/Jobs pattern), and layers on the +Models-specific ergonomics that used to live on the vendored Stainless +``ModelsResource``: + +- OpenAI inference-gateway route builders (``get_openai_route_base_url`` and + friends) -- pure string builders, safe from sync or async code, and +- deployment/provider status polling (``wait_for_deployment_status`` / + ``wait_for_provider_status``) driven by the client's own ``get_deployment`` / + ``get_provider`` methods. + +The inference-gateway *readiness* probe (``wait_for_gateway``) lives one layer +up in ``packages/models`` because it targets the separate inference-gateway +service, not Models -- see that module and AIRCORE notes. + +Usage:: + + from nemo_platform_plugin.models.client import ModelsClient + from nemo_platform_plugin.models.types import CreateModelEntityRequest + + client = ModelsClient(base_url="...", workspace="default") + model = client.create_model(body=CreateModelEntityRequest(name="llama")).data() + for m in client.list_models().items(): + print(m.name) + client.wait_for_deployment_status("my-deploy", "READY") +""" + +from __future__ import annotations + +import asyncio +import time +from datetime import datetime +from typing import Protocol + +from nemo_platform_plugin.client.client import AsyncNemoClient, NemoClient +from nemo_platform_plugin.client.errors import NotFoundError +from nemo_platform_plugin.client.method import method +from nemo_platform_plugin.models import endpoints +from nemo_platform_plugin.models.types import ModelDeployment + +_INFERENCE_GATEWAY_PREFIX = "/apis/inference-gateway/v2/workspaces" + + +# The OpenAI-route builders only read a couple of attributes, so they accept any +# object exposing them -- the plugin ``ModelProvider`` / ``ModelEntity`` models, +# or the Stainless SDK equivalents that ``packages/models`` passes through. +# Structural typing keeps the plugin free of a dependency on the generated SDK +# types while still accepting them. + + +class ProviderLike(Protocol): + """An object identifying a model provider and its upstream host URL.""" + + @property + def workspace(self) -> str: ... + @property + def name(self) -> str: ... + @property + def host_url(self) -> str: ... + + +class ModelEntityLike(Protocol): + """An object identifying a model entity.""" + + @property + def workspace(self) -> str: ... + @property + def name(self) -> str: ... + + +class DeploymentLike(Protocol): + """An object identifying a deployment and its auto-created provider.""" + + @property + def name(self) -> str: ... + @property + def model_provider_id(self) -> str | None: ... + + +def _seconds_since_creation(entry_timestamp: datetime | str | None, created_at: datetime | None) -> int | None: + """Seconds from deployment creation to the entry timestamp, or None if not comparable.""" + if created_at is None or entry_timestamp is None: + return None + if isinstance(entry_timestamp, str): + try: + entry_timestamp = datetime.fromisoformat(entry_timestamp.replace("Z", "+00:00")) + except (ValueError, TypeError): + return None + if not hasattr(entry_timestamp, "timestamp") or not hasattr(created_at, "timestamp"): + return None + try: + return int(entry_timestamp.timestamp() - created_at.timestamp()) + except (TypeError, OSError): + return None + + +def _deployment_status(deployment: ModelDeployment) -> tuple[str, str]: + """Return ``(current_status, status_message)`` for a deployment. + + The API guarantees the last history entry is the current state; fall back to + the top-level fields when there is no history. + """ + history = deployment.status_history + if history: + last = history[-1] + return last.status.value, last.status_message or "" + return deployment.status.value, deployment.status_message or "" + + +def _print_new_history(deployment: ModelDeployment, last_history_len: int) -> int: + """Print any status-history entries not yet seen; return the new history length.""" + history = deployment.status_history + created_at = deployment.created_at + if len(history) > last_history_len: + for entry in history[last_history_len:]: + ts = entry.timestamp + ts_str = ts.strftime("%H:%M:%S") if hasattr(ts, "strftime") else str(ts) + secs = _seconds_since_creation(ts, created_at) + part = f" [{ts_str}] " + if secs is not None: + part += f"(+{secs}s) " + part += f"Status: {entry.status.value}" + if entry.status_message: + part += f" - {entry.status_message}" + print(part) + return len(history) + return last_history_len + + +class _ModelsMethods: + # Model entities + create_model = method(endpoints.create_model) + list_models = method(endpoints.list_models) + get_model = method(endpoints.get_model) + update_model = method(endpoints.update_model) + delete_model = method(endpoints.delete_model) + + # Nested adapters (base model in the path) + create_model_adapter = method(endpoints.create_model_adapter) + update_model_adapter = method(endpoints.update_model_adapter) + delete_model_adapter = method(endpoints.delete_model_adapter) + + # Top-level adapters + create_adapter = method(endpoints.create_adapter) + list_adapters = method(endpoints.list_adapters) + get_adapter = method(endpoints.get_adapter) + update_adapter = method(endpoints.update_adapter) + delete_adapter = method(endpoints.delete_adapter) + + # Model providers + create_provider = method(endpoints.create_provider) + list_providers = method(endpoints.list_providers) + get_provider = method(endpoints.get_provider) + upsert_provider = method(endpoints.upsert_provider) + update_provider_status = method(endpoints.update_provider_status) + delete_provider = method(endpoints.delete_provider) + + # Prompts + create_prompt = method(endpoints.create_prompt) + list_prompts = method(endpoints.list_prompts) + get_prompt = method(endpoints.get_prompt) + update_prompt = method(endpoints.update_prompt) + delete_prompt = method(endpoints.delete_prompt) + + # Model deployments + create_deployment = method(endpoints.create_deployment) + list_deployments = method(endpoints.list_deployments) + get_deployment = method(endpoints.get_deployment) + get_deployment_models = method(endpoints.get_deployment_models) + list_deployment_versions = method(endpoints.list_deployment_versions) + get_deployment_version = method(endpoints.get_deployment_version) + update_deployment = method(endpoints.update_deployment) + update_deployment_status = method(endpoints.update_deployment_status) + delete_deployment = method(endpoints.delete_deployment) + delete_deployment_version = method(endpoints.delete_deployment_version) + + # Model deployment configs + create_deployment_config = method(endpoints.create_deployment_config) + list_deployment_configs = method(endpoints.list_deployment_configs) + get_deployment_config = method(endpoints.get_deployment_config) + list_deployment_config_versions = method(endpoints.list_deployment_config_versions) + get_deployment_config_version = method(endpoints.get_deployment_config_version) + update_deployment_config = method(endpoints.update_deployment_config) + delete_deployment_config = method(endpoints.delete_deployment_config) + delete_deployment_config_version = method(endpoints.delete_deployment_config_version) + + +class _ModelsUrlMixin: + """Pure OpenAI-route URL builders. No I/O -- safe from sync or async code. + + Depends only on the client's ``base_url`` and default ``workspace`` (both + provided by :class:`BaseNemoClient`). + """ + + base_url: str + workspace: str | None + + def _resolve_workspace(self, workspace: str | None) -> str: + ws = workspace or self.workspace + if not ws: + raise ValueError("Missing workspace argument; either set a client-level workspace or pass workspace=...") + return ws + + def get_openai_route_base_url(self, *, workspace: str | None = None) -> str: + """Base URL for the OpenAI proxy route (routes on the request body ``model`` field).""" + ws = self._resolve_workspace(workspace) + return f"{self.base_url}/{_INFERENCE_GATEWAY_PREFIX.lstrip('/')}/{ws}/openai/-/v1" + + def get_provider_route_openai_url(self, provider: ProviderLike) -> str: + """OpenAI SDK-compatible URL for a provider proxy route. + + Appends ``/v1`` unless the provider's ``host_url`` already ends in ``/v1``. + """ + route = ( + f"{self.base_url}/{_INFERENCE_GATEWAY_PREFIX.lstrip('/')}/{provider.workspace}/provider/{provider.name}/-" + ) + if not provider.host_url.rstrip("/").endswith("/v1"): + route = f"{route}/v1" + return route + + def get_model_entity_route_openai_url(self, model_entity: ModelEntityLike) -> str: + """OpenAI SDK-compatible URL for a model-entity proxy route (always ``/v1``).""" + return ( + f"{self.base_url}/{_INFERENCE_GATEWAY_PREFIX.lstrip('/')}/" + f"{model_entity.workspace}/model/{model_entity.name}/-/v1" + ) + + +class ModelsClient(_ModelsMethods, _ModelsUrlMixin, NemoClient): + """Sync client for the Models service API.""" + + def get_provider_route_openai_url_for_deployment(self, deployment: DeploymentLike) -> str: + """Fetch a deployment's ModelProvider and return its OpenAI route URL.""" + if not deployment.model_provider_id: + raise ValueError(f"Deployment '{deployment.name}' has no associated model_provider_id") + workspace, name = deployment.model_provider_id.split("/", 1) + provider = self.get_provider(name=name, workspace=workspace).data() + return self.get_provider_route_openai_url(provider) + + def wait_for_deployment_status( + self, + deployment_name: str, + desired_status: str, + *, + workspace: str | None = None, + timeout: int = 1200, + poll_interval: float = 3.0, + ) -> bool: + """Poll a ModelDeployment until it reaches ``desired_status`` (or times out). + + For ``"DELETED"``, waits for the resource to be fully garbage collected + (404), not merely for the status to read DELETED. Returns False on + timeout or a terminal ERROR state. + """ + start = time.time() + last_status = "" + last_message = "" + last_history_len = 0 + print(f"Waiting for status: {desired_status}...\n") + + while time.time() - start < timeout: + try: + deployment = self.get_deployment(name=deployment_name, workspace=workspace).data() + except NotFoundError: + if desired_status == "DELETED": + print(f"Deployment {desired_status}!\n") + return True + print("Deployment not found\n") + return False + + current_status, status_message = _deployment_status(deployment) + last_status, last_message = current_status, status_message + last_history_len = _print_new_history(deployment, last_history_len) + + if current_status == desired_status and desired_status != "DELETED": + print(f"Deployment reached {desired_status} status!\n") + return True + if current_status == "ERROR": + print(f"Deployment entered ERROR state: {status_message}\n") + return False + time.sleep(poll_interval) + + detail = f"Last status: {last_status}" + if last_message: + detail += f" - {last_message}" + print(f"Timeout after {int(time.time() - start)}s. {detail}\n") + return False + + def wait_for_provider_status( + self, + provider_name: str, + desired_status: str = "READY", + *, + workspace: str | None = None, + timeout: int = 60, + poll_interval: float = 1.0, + ) -> bool: + """Poll a ModelProvider until it reaches ``desired_status`` (or times out).""" + start = time.time() + last_status = "" + print(f"Waiting for provider '{provider_name}' to reach status: {desired_status}...") + + while time.time() - start < timeout: + try: + provider = self.get_provider(name=provider_name, workspace=workspace).data() + except NotFoundError: + print(f"\nProvider '{provider_name}' not found\n") + return False + + current_status = provider.status.value + if current_status != last_status: + elapsed = int(time.time() - start) + print(f" [{datetime.now().strftime('%H:%M:%S')}] ({elapsed}s) Status: {current_status}") + last_status = current_status + if current_status == desired_status: + return True + if current_status == "ERROR": + print(f"\nProvider entered ERROR state: {provider.status_message}\n") + return False + time.sleep(poll_interval) + + print(f"\nProvider timeout after {int(time.time() - start)}s. Last status: {last_status}\n") + return False + + +class AsyncModelsClient(_ModelsMethods, _ModelsUrlMixin, AsyncNemoClient): + """Async client for the Models service API.""" + + async def get_provider_route_openai_url_for_deployment(self, deployment: DeploymentLike) -> str: + """Fetch a deployment's ModelProvider and return its OpenAI route URL.""" + if not deployment.model_provider_id: + raise ValueError(f"Deployment '{deployment.name}' has no associated model_provider_id") + workspace, name = deployment.model_provider_id.split("/", 1) + provider = (await self.get_provider(name=name, workspace=workspace)).data() + return self.get_provider_route_openai_url(provider) + + async def wait_for_deployment_status( + self, + deployment_name: str, + desired_status: str, + *, + workspace: str | None = None, + timeout: int = 1200, + poll_interval: float = 3.0, + ) -> bool: + """Async twin of :meth:`ModelsClient.wait_for_deployment_status`.""" + start = time.time() + last_status = "" + last_message = "" + last_history_len = 0 + print(f"Waiting for status: {desired_status}...\n") + + while time.time() - start < timeout: + try: + deployment = (await self.get_deployment(name=deployment_name, workspace=workspace)).data() + except NotFoundError: + if desired_status == "DELETED": + print(f"Deployment {desired_status}!\n") + return True + print("Deployment not found\n") + return False + + current_status, status_message = _deployment_status(deployment) + last_status, last_message = current_status, status_message + last_history_len = _print_new_history(deployment, last_history_len) + + if current_status == desired_status and desired_status != "DELETED": + print(f"Deployment reached {desired_status} status!\n") + return True + if current_status == "ERROR": + print(f"Deployment entered ERROR state: {status_message}\n") + return False + await asyncio.sleep(poll_interval) + + detail = f"Last status: {last_status}" + if last_message: + detail += f" - {last_message}" + print(f"Timeout after {int(time.time() - start)}s. {detail}\n") + return False + + async def wait_for_provider_status( + self, + provider_name: str, + desired_status: str = "READY", + *, + workspace: str | None = None, + timeout: int = 60, + poll_interval: float = 1.0, + ) -> bool: + """Async twin of :meth:`ModelsClient.wait_for_provider_status`.""" + start = time.time() + last_status = "" + print(f"Waiting for provider '{provider_name}' to reach status: {desired_status}...") + + while time.time() - start < timeout: + try: + provider = (await self.get_provider(name=provider_name, workspace=workspace)).data() + except NotFoundError: + print(f"\nProvider '{provider_name}' not found\n") + return False + + current_status = provider.status.value + if current_status != last_status: + elapsed = int(time.time() - start) + print(f" [{datetime.now().strftime('%H:%M:%S')}] ({elapsed}s) Status: {current_status}") + last_status = current_status + if current_status == desired_status: + return True + if current_status == "ERROR": + print(f"\nProvider entered ERROR state: {provider.status_message}\n") + return False + await asyncio.sleep(poll_interval) + + print(f"\nProvider timeout after {int(time.time() - start)}s. Last status: {last_status}\n") + return False diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/endpoints.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/endpoints.py new file mode 100644 index 0000000000..ab1916510d --- /dev/null +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/endpoints.py @@ -0,0 +1,390 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Typed endpoint definitions for the Models service. + +These are the single source of truth for the HTTP contract. All paths include +the ``/apis/models`` gateway prefix. Although the Stainless SDK grouped some of +these under ``sdk.inference.*`` (deployments, providers, prompts), every route +is served by the Models service under ``/apis/models/v2/...``. + +The service exposes six resource groups: +- model entities (``/models``) and their nested adapters (``/models/{m}/adapters``), +- top-level adapters (``/adapters``), +- model providers (``/providers``), +- prompts (``/prompts``), +- model deployments (``/deployments``) with immutable versioning, and +- model deployment configs (``/deployment-configs``) with immutable versioning. +""" + +from __future__ import annotations + +from abc import abstractmethod +from typing import Any + +from nemo_platform_plugin.client.endpoint import delete, get, patch, post, put +from nemo_platform_plugin.client.types import Paginated, PreparedRequest +from nemo_platform_plugin.models.types import ( + Adapter, + CreateAdapterRequest, + CreateModelAdapterRequest, + CreateModelDeploymentConfigRequest, + CreateModelDeploymentRequest, + CreateModelEntityRequest, + CreateModelProviderRequest, + CreatePromptRequest, + GetModelQueryParams, + ListAdaptersQueryParams, + ListDeploymentConfigsQueryParams, + ListDeploymentsQueryParams, + ListModelsQueryParams, + ListPromptsQueryParams, + ListProvidersQueryParams, + ModelDeployment, + ModelDeploymentConfig, + ModelEntity, + ModelProvider, + Prompt, + UpdateAdapterRequest, + UpdateDeploymentStatusQueryParams, + UpdateModelDeploymentConfigRequest, + UpdateModelDeploymentRequest, + UpdateModelDeploymentStatusRequest, + UpdateModelEntityRequest, + UpdateModelProviderStatusRequest, + UpdatePromptRequest, + UpsertModelProviderRequest, +) + +_MODELS = "/apis/models/v2/workspaces/{workspace}" + + +# --------------------------------------------------------------------------- +# Model entities +# --------------------------------------------------------------------------- + + +@get(_MODELS + "/models/{name}") +@abstractmethod +def get_model( + *, workspace: str | None = None, name: str, query_params: GetModelQueryParams | None = None +) -> ModelEntity: ... + + +@get(_MODELS + "/models") +@abstractmethod +def list_models( + *, workspace: str | None = None, query_params: ListModelsQueryParams | None = None +) -> Paginated[ModelEntity]: ... + + +def _get_model_on_conflict(body: CreateModelEntityRequest, workspace: str | None) -> PreparedRequest[ModelEntity]: + """Retrieve request replayed when ``create_model(exist_ok=True)`` 409s.""" + return get_model(name=body.name, workspace=workspace) + + +@post(_MODELS + "/models", get_on_conflict=_get_model_on_conflict) +@abstractmethod +def create_model( + *, workspace: str | None = None, body: CreateModelEntityRequest, exist_ok: bool = False +) -> ModelEntity: ... + + +@patch(_MODELS + "/models/{name}") +@abstractmethod +def update_model( + *, + workspace: str | None = None, + name: str, + body: UpdateModelEntityRequest, + query_params: GetModelQueryParams | None = None, +) -> ModelEntity: ... + + +@delete(_MODELS + "/models/{name}") +@abstractmethod +def delete_model(*, workspace: str | None = None, name: str) -> None: ... + + +# --------------------------------------------------------------------------- +# Nested adapters (base model in the path) +# --------------------------------------------------------------------------- + + +@post(_MODELS + "/models/{model_name}/adapters") +@abstractmethod +def create_model_adapter( + *, workspace: str | None = None, model_name: str, body: CreateModelAdapterRequest +) -> Adapter: ... + + +@patch(_MODELS + "/models/{model_name}/adapters/{adapter}") +@abstractmethod +def update_model_adapter( + *, workspace: str | None = None, model_name: str, adapter: str, body: UpdateAdapterRequest +) -> Adapter: ... + + +@delete(_MODELS + "/models/{model_name}/adapters/{adapter}") +@abstractmethod +def delete_model_adapter(*, workspace: str | None = None, model_name: str, adapter: str) -> None: ... + + +# --------------------------------------------------------------------------- +# Top-level adapters +# --------------------------------------------------------------------------- + + +@get(_MODELS + "/adapters/{name}") +@abstractmethod +def get_adapter(*, workspace: str | None = None, name: str) -> Adapter: ... + + +@get(_MODELS + "/adapters") +@abstractmethod +def list_adapters( + *, workspace: str | None = None, query_params: ListAdaptersQueryParams | None = None +) -> Paginated[Adapter]: ... + + +def _get_adapter_on_conflict(body: CreateAdapterRequest, workspace: str | None) -> PreparedRequest[Adapter]: + return get_adapter(name=body.name, workspace=workspace) + + +@post(_MODELS + "/adapters", get_on_conflict=_get_adapter_on_conflict) +@abstractmethod +def create_adapter(*, workspace: str | None = None, body: CreateAdapterRequest, exist_ok: bool = False) -> Adapter: ... + + +@patch(_MODELS + "/adapters/{name}") +@abstractmethod +def update_adapter(*, workspace: str | None = None, name: str, body: UpdateAdapterRequest) -> Adapter: ... + + +@delete(_MODELS + "/adapters/{name}") +@abstractmethod +def delete_adapter(*, workspace: str | None = None, name: str) -> None: ... + + +# --------------------------------------------------------------------------- +# Model providers +# --------------------------------------------------------------------------- + + +@get(_MODELS + "/providers/{name}") +@abstractmethod +def get_provider(*, workspace: str | None = None, name: str) -> ModelProvider: ... + + +@get(_MODELS + "/providers") +@abstractmethod +def list_providers( + *, workspace: str | None = None, query_params: ListProvidersQueryParams | None = None +) -> Paginated[ModelProvider]: ... + + +def _get_provider_on_conflict( + body: CreateModelProviderRequest, workspace: str | None +) -> PreparedRequest[ModelProvider]: + return get_provider(name=body.name, workspace=workspace) + + +@post(_MODELS + "/providers", get_on_conflict=_get_provider_on_conflict) +@abstractmethod +def create_provider( + *, workspace: str | None = None, body: CreateModelProviderRequest, exist_ok: bool = False +) -> ModelProvider: ... + + +@put(_MODELS + "/providers/{name}") +@abstractmethod +def upsert_provider(*, workspace: str | None = None, name: str, body: UpsertModelProviderRequest) -> ModelProvider: ... + + +@put(_MODELS + "/providers/{name}/status") +@abstractmethod +def update_provider_status( + *, workspace: str | None = None, name: str, body: UpdateModelProviderStatusRequest +) -> ModelProvider: ... + + +@delete(_MODELS + "/providers/{name}") +@abstractmethod +def delete_provider(*, workspace: str | None = None, name: str) -> None: ... + + +# --------------------------------------------------------------------------- +# Prompts +# --------------------------------------------------------------------------- + + +@get(_MODELS + "/prompts/{name}") +@abstractmethod +def get_prompt(*, workspace: str | None = None, name: str) -> Prompt: ... + + +@get(_MODELS + "/prompts") +@abstractmethod +def list_prompts( + *, workspace: str | None = None, query_params: ListPromptsQueryParams | None = None +) -> Paginated[Prompt]: ... + + +def _get_prompt_on_conflict(body: CreatePromptRequest, workspace: str | None) -> PreparedRequest[Prompt]: + return get_prompt(name=body.name, workspace=workspace) + + +@post(_MODELS + "/prompts", get_on_conflict=_get_prompt_on_conflict) +@abstractmethod +def create_prompt(*, workspace: str | None = None, body: CreatePromptRequest, exist_ok: bool = False) -> Prompt: ... + + +@put(_MODELS + "/prompts/{name}") +@abstractmethod +def update_prompt(*, workspace: str | None = None, name: str, body: UpdatePromptRequest) -> Prompt: ... + + +@delete(_MODELS + "/prompts/{name}") +@abstractmethod +def delete_prompt(*, workspace: str | None = None, name: str) -> None: ... + + +# --------------------------------------------------------------------------- +# Model deployments +# --------------------------------------------------------------------------- + + +@get(_MODELS + "/deployments/{name}") +@abstractmethod +def get_deployment(*, workspace: str | None = None, name: str) -> ModelDeployment: ... + + +@get(_MODELS + "/deployments") +@abstractmethod +def list_deployments( + *, workspace: str | None = None, query_params: ListDeploymentsQueryParams | None = None +) -> Paginated[ModelDeployment]: ... + + +@get(_MODELS + "/deployments/{name}/models") +@abstractmethod +def get_deployment_models(*, workspace: str | None = None, name: str) -> dict[str, Any]: ... + + +@get(_MODELS + "/deployments/{name}/versions") +@abstractmethod +def list_deployment_versions(*, workspace: str | None = None, name: str) -> list[ModelDeployment]: ... + + +@get(_MODELS + "/deployments/{deployment}/versions/{name}") +@abstractmethod +def get_deployment_version(*, workspace: str | None = None, deployment: str, name: str) -> ModelDeployment: ... + + +def _get_deployment_on_conflict( + body: CreateModelDeploymentRequest, workspace: str | None +) -> PreparedRequest[ModelDeployment]: + return get_deployment(name=body.name, workspace=workspace) + + +@post(_MODELS + "/deployments", get_on_conflict=_get_deployment_on_conflict) +@abstractmethod +def create_deployment( + *, workspace: str | None = None, body: CreateModelDeploymentRequest, exist_ok: bool = False +) -> ModelDeployment: ... + + +@post(_MODELS + "/deployments/{name}") +@abstractmethod +def update_deployment( + *, workspace: str | None = None, name: str, body: UpdateModelDeploymentRequest +) -> ModelDeployment: ... + + +@post(_MODELS + "/deployments/{name}/status") +@abstractmethod +def update_deployment_status( + *, + workspace: str | None = None, + name: str, + body: UpdateModelDeploymentStatusRequest, + query_params: UpdateDeploymentStatusQueryParams | None = None, +) -> ModelDeployment: ... + + +@delete(_MODELS + "/deployments/{name}") +@abstractmethod +def delete_deployment(*, workspace: str | None = None, name: str) -> None: + """Delete a deployment. + + Returns 202 Accepted when teardown is asynchronous (the deployment enters + DELETING while infrastructure is torn down) or 204 No Content when the + delete is synchronous (already hard-deleted). Both are success and the typed + body is ``None``; read ``resp.http_response.status_code`` to distinguish them. + """ + + +@delete(_MODELS + "/deployments/{deployment}/versions/{name}") +@abstractmethod +def delete_deployment_version(*, workspace: str | None = None, deployment: str, name: str) -> None: + """Delete a single deployment version (202 async / 204 synchronous; body ``None``). + + See :func:`delete_deployment` for the 202-vs-204 distinction. + """ + + +# --------------------------------------------------------------------------- +# Model deployment configs +# --------------------------------------------------------------------------- + + +@get(_MODELS + "/deployment-configs/{name}") +@abstractmethod +def get_deployment_config(*, workspace: str | None = None, name: str) -> ModelDeploymentConfig: ... + + +@get(_MODELS + "/deployment-configs") +@abstractmethod +def list_deployment_configs( + *, workspace: str | None = None, query_params: ListDeploymentConfigsQueryParams | None = None +) -> Paginated[ModelDeploymentConfig]: ... + + +@get(_MODELS + "/deployment-configs/{name}/versions") +@abstractmethod +def list_deployment_config_versions(*, workspace: str | None = None, name: str) -> list[ModelDeploymentConfig]: ... + + +@get(_MODELS + "/deployment-configs/{config}/versions/{name}") +@abstractmethod +def get_deployment_config_version(*, workspace: str | None = None, config: str, name: str) -> ModelDeploymentConfig: ... + + +def _get_deployment_config_on_conflict( + body: CreateModelDeploymentConfigRequest, workspace: str | None +) -> PreparedRequest[ModelDeploymentConfig]: + return get_deployment_config(name=body.name, workspace=workspace) + + +@post(_MODELS + "/deployment-configs", get_on_conflict=_get_deployment_config_on_conflict) +@abstractmethod +def create_deployment_config( + *, workspace: str | None = None, body: CreateModelDeploymentConfigRequest, exist_ok: bool = False +) -> ModelDeploymentConfig: ... + + +@post(_MODELS + "/deployment-configs/{name}") +@abstractmethod +def update_deployment_config( + *, workspace: str | None = None, name: str, body: UpdateModelDeploymentConfigRequest +) -> ModelDeploymentConfig: ... + + +@delete(_MODELS + "/deployment-configs/{name}") +@abstractmethod +def delete_deployment_config(*, workspace: str | None = None, name: str) -> None: ... + + +@delete(_MODELS + "/deployment-configs/{config}/versions/{name}") +@abstractmethod +def delete_deployment_config_version(*, workspace: str | None = None, config: str, name: str) -> None: ... diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/types.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/types.py new file mode 100644 index 0000000000..0ce274132d --- /dev/null +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/types.py @@ -0,0 +1,1466 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Typed request/response models for the Models service client. + +These models mirror the HTTP contract for model entities, adapters, model +providers, prompts, model deployments, and model deployment configs. The +Models service remains the authoritative wire-schema owner; this plugin module +keeps client DTOs independent of the Stainless-generated SDK. + +The plugin package must stay free of an ``nmp_common`` dependency because that +would create a reverse service dependency. Server-only pieces are handled per +the data-vs-behavior split documented in ``client/MIGRATION.md``: + +- **Constants** (name regex, max lengths) that live in + ``nmp.common.entities.constants`` are inlined here with a comment pointing at + the origin (matching the ``secrets.types`` boundary). +- **``AuthContext``** is a pure-data mirror of ``nmp.common.auth.AuthContext`` + with the same wire shape and no ``from_principal``/``to_principal`` behavior. +- **``InferenceParams``** is a faithful replica of + ``nmp.common.inference.InferenceParams`` (pure pydantic, no server imports). +- **``BackendFormat``** is reused from ``nemo_platform_plugin.inference_middleware``. +- The Jinja2 ``auth_header_format`` validator needs ``jinja2`` (not a plugin + dependency) so it stays server-side; the plugin field is a plain string. +- Entity-store ``Filter`` subclasses are genuinely server-only and are not + mirrored here; the client passes filters as a ``filter`` query-param string. +""" + +from __future__ import annotations + +import re +from datetime import datetime +from enum import Enum, StrEnum +from typing import Any, NotRequired, Self, TypedDict + +from nemo_platform_plugin.inference_middleware import BackendFormat +from pydantic import AnyUrl, BaseModel, ConfigDict, Field, field_validator, model_validator + +# --------------------------------------------------------------------------- +# Inlined constants +# +# Mirror ``nmp.common.entities.constants`` and ``nmp.core.models.constants``. +# Inlined rather than imported so this package stays free of an ``nmp_common`` +# (or models-service) dependency -- see the module docstring. +# --------------------------------------------------------------------------- + +_NAME_REGEX = r"^[\w\-.]+$" # constants.REGEX_WORD_CHARACTER_DOT_DASH +_NAME_SLASH_REGEX = r"^[\w\-./]+$" # constants.REGEX_WORD_CHARACTER_DOT_DASH_SLASH +_NAME_DESC = "Allowed characters: letters (a-z, A-Z), digits (0-9), underscores, hyphens, and dots." +_MAX_LEN_255 = 255 # constants.MAX_LENGTH_255 + +# nmp.core.models.constants -- adapter/model reference rules. +_MODEL_REF_NAME_SEGMENT = r"[a-z](?!.*--)[a-z0-9\-@.+_]{1,62}(? bool: + """True if *value* matches :data:`MODEL_REF_PATTERN` (entity NAME rules per segment).""" + return _MODEL_REF_RE.fullmatch(value) is not None + + +# --------------------------------------------------------------------------- +# Auth context (data-only mirror of nmp.common.auth.AuthContext) +# --------------------------------------------------------------------------- + + +class AuthContext(BaseModel): + """Auth context captured at resource creation for delegated access. + + This is the wire/data shape. The server's ``nmp.common.auth.AuthContext`` + adds ``from_principal`` / ``to_principal`` behavior on top of the same + fields, and the server response models re-type this field to that class. + """ + + principal_id: str = Field(..., description="The principal's unique identifier") + principal_email: str | None = Field(default=None, description="The principal's email address") + principal_groups: list[str] = Field(default_factory=list, description="Groups the principal belongs to") + principal_on_behalf_of: str | None = Field( + default=None, description="If acting on behalf of another principal, their principal ID" + ) + principal_on_behalf_of_groups: list[str] | None = Field( + default=None, description="Groups the on-behalf-of principal belongs to" + ) + principal_on_behalf_of_email: str | None = Field( + default=None, description="The on-behalf-of principal's email address" + ) + + +# --------------------------------------------------------------------------- +# Inference parameters (replica of nmp.common.inference.InferenceParams) +# --------------------------------------------------------------------------- + + +class InferenceParams(BaseModel): + """Parameters for model inference. + + Extra fields can be supplied for additional options applied to the inference + request directly. Fields not supported by the model may cause inference + errors during evaluation. + """ + + model_config = ConfigDict(extra="allow") + + model: str | None = Field(default=None, description="Model identifier") + temperature: float | None = Field( + default=None, + ge=0, + le=2, + description="Float value between 0 and 1. temp of 0 indicates greedy decoding, " + "where the token with highest prob is chosen. Temperature can't be set to 0.0 currently", + ) + max_tokens: int | None = Field(default=None, ge=1, description="Max tokens to generate") + max_completion_tokens: int | None = Field(default=None, ge=1, description="Max tokens to generate") + top_p: float | None = Field( + default=None, + ge=0, + le=1, + description="Float value between 0 and 1; limits to the top tokens within a certain " + "probability. top_p=0 means the model will only consider the single most likely " + "token for the next prediction", + ) + stop: list[str] | None = Field(default=None) + + @model_validator(mode="after") + def check_max_tokens(self) -> Self: + if self.max_tokens and self.max_completion_tokens: + raise ValueError( + "max_tokens and max_completion_tokens cannot both be configured. " + "Choose the appropriate tokens parameter for the model." + ) + return self + + +# --------------------------------------------------------------------------- +# Value types +# --------------------------------------------------------------------------- + + +class ModelPrecision(str, Enum): + """Type of model precision.""" + + INT8 = "int8" + BF16 = "bf16" + FP16 = "fp16" + FP32 = "fp32" + FP8_MIXED = "fp8-mixed" + BF16_MIXED = "bf16-mixed" + + +class FinetuningType(str, Enum): + """Finetuning types.""" + + LORA_MERGED = "lora_merged" + ALL_WEIGHTS = "all_weights" + + LAST_LAYER = "last_layer" + TOP_LAYERS = "top_layers" + GRADUAL_UNFREEZING = "gradual_unfreezing" + BIAS_ONLY = "bias_only" # BitFit + ATTENTION_ONLY = "attention_only" + + LORA = "lora" + QLORA = "qlora" + ADALORA = "adalora" + DORA = "dora" + LORA_PLUS = "lora_plus" + + PROMPT_TUNING = "prompt_tuning" + PREFIX_TUNING = "prefix_tuning" + P_TUNING = "p_tuning" + P_TUNING_V2 = "p_tuning_v2" + SOFT_PROMPT = "soft_prompt" + + PPO = "ppo" + DPO = "dpo" + CDPO = "cdpo" + IPO = "ipo" + ORPO = "orpo" + KTO = "kto" + RRHF = "rrhf" + GRPO = "grpo" + + +class MoEConfig(BaseModel): + """Mixture of Experts configuration.""" + + num_experts: int = Field(description="Total number of routed experts (sharded by EP)") + num_experts_per_tok: int = Field(description="Number of experts activated per token (top-k routing)") + num_expert_layers: int = Field(description="Number of layers with MoE") + expert_ffn_size: int | None = Field(default=None, description="FFN size for experts (if different from main FFN)") + num_shared_experts: int = Field(default=0, description="Number of shared experts (replicated, not sharded by EP)") + + +class MambaConfig(BaseModel): + """Mamba/State Space Model configuration.""" + + is_hybrid: bool = Field(description="Whether model is Mamba-Transformer hybrid") + num_mamba_layers: int = Field(description="Number of Mamba/SSM layers") + num_attention_layers: int = Field(default=0, description="Number of attention layers (for hybrids)") + num_mlp_layers: int = Field( + default=0, description="Number of standalone MLP layers (for interleaved architectures)" + ) + state_size: int = Field(default=16, description="SSM state expansion factor (d_state)") + conv_kernel: int = Field(default=4, description="Convolution kernel size for Mamba (d_conv)") + + +class SlidingWindowConfig(BaseModel): + """Sliding window attention configuration.""" + + window_size: int = Field(description="Sliding window size (attends to last N tokens)") + + +class ToolCallConfig(BaseModel): + """Configuration for tool calling support in NIM deployments.""" + + tool_call_parser: str | None = Field( + default=None, + description="Name of the tool call parser to use (e.g., 'openai', 'hermes', 'pythonic', 'llama3_json', 'mistral').", + max_length=_MAX_LEN_255, + ) + tool_call_plugin: str | None = Field( + default=None, + description="Reference to a fileset containing the custom tool call plugin Python file. " + "Expected format: '{workspace}/{fileset_name}'. The fileset is mounted separately from " + "the model checkpoint at deployment time.", + max_length=_MAX_LEN_255, + ) + auto_tool_choice: bool | None = Field( + default=None, + description="Whether to enable automatic tool choice. When enabled, the model can decide to call tools " + "without explicit user instruction.", + ) + + +class LinearLayerSpec(BaseModel): + """Specification for a single linear layer in the model.""" + + name: str = Field(description="Module name (e.g., 'model.layers.0.self_attn.q_proj')") + in_features: int = Field(description="Input feature dimension") + out_features: int = Field(description="Output feature dimension") + + +class ModelSpec(BaseModel): + """Detailed specification for a model.""" + + context_size: int | None = Field(None, description="Context window size") + num_virtual_tokens: int | None = Field(None, description="Number of virtual tokens for prompt tuning") + is_chat: bool | None = Field(None, description="Whether this is a chat model") + is_embedding_model: bool = Field(False, description="Whether this is an embedding model") + + # Basic model information + checkpoint_model_name: str = Field(description="Checkpoint Model identifier or model path") + family: str = Field(description="Model architecture family (e.g., 'llama', 'mixtral', 'gpt2')") + + # Architecture dimensions + num_layers: int = Field(description="Number of transformer layers") + hidden_size: int = Field(description="Hidden dimension size") + num_attention_heads: int = Field(description="Number of attention heads") + num_kv_heads: int = Field(description="Number of key-value heads (for GQA/MQA)") + ffn_hidden_size: int = Field(description="FFN intermediate size") + vocab_size: int = Field(description="Vocabulary size") + + # Model properties + tied_embeddings: bool = Field(description="Whether embeddings are tied") + gated_mlp: bool = Field(description="Whether MLP uses gated activation") + base_num_parameters: int = Field(description="Total model parameters") + precision: str = Field(description="Model precision (e.g., 'float16', 'bfloat16', 'float32', 'int8', 'int4')") + + # Optional configurations + moe_config: MoEConfig | None = Field(default=None, description="MoE configuration if applicable") + mamba_config: MambaConfig | None = Field(default=None, description="Mamba/SSM configuration if applicable") + sliding_window_config: SlidingWindowConfig | None = Field( + default=None, description="Sliding window attention config if applicable" + ) + + # LoRA-specific metadata (pre-computed to avoid model instantiation) + linear_layers: list[LinearLayerSpec] | None = Field( + default=None, + description="List of all linear/Conv1D layers with their dimensions. " + "Used for LoRA parameter estimation without requiring model instantiation. " + "Each entry contains the module name, in_features, and out_features.", + ) + + # Deployment configuration + chat_template: str | None = Field( + default=None, + description="Jinja2 chat template string for the model. Used by NIM to format chat completions. " + "If not set, the model's built-in tokenizer template is used.", + ) + tool_call_config: ToolCallConfig | None = Field( + default=None, + description="Tool calling configuration for NIM deployments. Controls how the model handles " + "function/tool calling in chat completions.", + ) + + # GPU requirements (auto-calculated) + minimum_gpus_all_weights: int | None = Field( + default=None, + description="Minimum GPUs required for full fine-tuning using default configurations.", + ) + minimum_gpus_lora: int | None = Field( + default=None, + description="Minimum GPUs required for LoRA fine-tuning using default configurations.", + ) + + def model_precision(self) -> ModelPrecision: + """Convert the precision string to a :class:`ModelPrecision` enum.""" + precision_map = { + "bf16-mixed": ModelPrecision.BF16_MIXED, + "bf16": ModelPrecision.BF16, + "bfloat16": ModelPrecision.BF16, + "float16": ModelPrecision.FP16, + "float32": ModelPrecision.FP32, + "fp16": ModelPrecision.FP16, + "fp32": ModelPrecision.FP32, + "fp8-mixed": ModelPrecision.FP8_MIXED, + "int4": ModelPrecision.INT8, # Map int4 to int8 as int4 is not in the enum + "int8": ModelPrecision.INT8, + } + if self.precision in precision_map: + return precision_map[self.precision] + return ModelPrecision.BF16 + + +class Lora(BaseModel): + alpha: int | None = Field(None, description="Alpha scaling used for this adapter") + rank: int = Field(..., description="LoRA Rank") + + +class APIEndpointData(BaseModel): + """Data about an inference endpoint.""" + + url: AnyUrl | None = Field(None, description="Endpoint URL") + model_id: str | None = Field(None, description="Model identifier at the endpoint") + api_key: str | None = Field(None, description="API key for authentication") + format: str | None = Field(None, description="API format (e.g., openai, nvidia)") + + +class PromptData(BaseModel): + """Configuration for prompt engineering.""" + + system_prompt: str | None = Field(None, description="System prompt template") + icl_few_shot_examples: str | None = Field(None, description="In-context learning examples") + inference_params: InferenceParams | None = Field( + default=None, description="Inference parameters that should be overridden." + ) + system_prompt_template: str | None = Field( + default=None, + title="System Prompt Template", + description="The template which will be used to compile the final prompt used for prompting the LLM. Currently supports only {{icl_few_shot_examples}}", + ) + + +# --------------------------------------------------------------------------- +# Base model +# --------------------------------------------------------------------------- + + +class ModelEntityBaseModel(BaseModel): + """Base model for all Models service domain objects.""" + + id: str = Field(..., description="Autogenerated id") + name: str = Field( + description=f"Name of the entity. Name/workspace combo must be unique across all entities. {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + examples=["llama-3.1-8b", "my-custom-model"], + ) + workspace: str = Field( + description=f"The workspace of the entity. {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + ) + project: str | None = Field( + default=None, + description="The URN of the project associated with this entity.", + max_length=_MAX_LEN_255, + pattern=_NAME_SLASH_REGEX, + ) + created_at: datetime = Field(..., description="The timestamp of model entity creation") + updated_at: datetime = Field(..., description="The timestamp of the last model entity update") + + +# --------------------------------------------------------------------------- +# ModelProvider +# --------------------------------------------------------------------------- + +_AUTH_HEADER_FORMAT_DESCRIPTION = ( + "Jinja2 template string controlling how the API key secret is sent to the upstream. " + "Must contain exactly one variable named `auth_secret`, which is substituted with the " + "resolved secret value at request time. " + "Example: `'X-Api-Key: {{ auth_secret }}'`. " + "If not set, defaults to `'Authorization: Bearer {{ auth_secret }}'`." +) + + +class ModelProviderStatus(str, Enum): + """Status enum for ModelProvider objects.""" + + UNKNOWN = "UNKNOWN" + CREATED = "CREATED" + PENDING = "PENDING" + READY = "READY" + ERROR = "ERROR" + DELETING = "DELETING" + DELETED = "DELETED" + LOST = "LOST" + + +class ServedModelMapping(BaseModel): + """Mapping between a Model Entity and how it's served by this provider.""" + + model_entity_id: str = Field( + description="Model Entity identifier as workspace/name (e.g., 'my-ws/my-model')", + max_length=_MAX_LEN_255, + ) + served_model_name: str = Field( + description="The actual model name to send to the backend endpoint in the 'model' field", + max_length=_MAX_LEN_255, + ) + + +class ModelProvider(ModelEntityBaseModel): + """A reachable network endpoint that provides inference for one or more Model Entities. + + The unique identifier for a ModelProvider is the combination of workspace/name. + """ + + id: str = Field(default="", description="Unique identifier for the model provider") + description: str | None = Field( + default=None, + description="Optional description of the model provider", + max_length=1000, + ) + host_url: str = Field( + description="The network endpoint URL for the model provider", + max_length=2048, + ) + api_key_secret_name: str | None = Field( + default=None, + description="Reference to the API key stored in Secrets service", + max_length=_MAX_LEN_255, + ) + served_models: list[ServedModelMapping] | None = Field( + default_factory=list, + description="List of models served by this provider with routing information for IGW", + ) + enabled_models: list[str] | None = Field( + default=None, + description="Optional list of specific models to enable from this provider. If not set, all discovered models are enabled.", + ) + status: ModelProviderStatus = Field( + default=ModelProviderStatus.UNKNOWN, + description="Current status of the model provider, populated by models service", + ) + status_message: str = Field( + default="", + description="Detailed status message, populated by models service", + max_length=1000, + ) + default_extra_body: dict[str, Any] | None = Field( + default=None, + description="Default body parameters for inference requests. Can be overridden by user requests.", + ) + default_extra_headers: dict[str, str] | None = Field( + default=None, + description="Default headers for inference requests. Can be overridden by user requests.", + ) + required_extra_body: dict[str, Any] | None = Field( + default=None, + description="Required body parameters for inference requests. Cannot be overridden by user requests.", + ) + required_extra_headers: dict[str, str] | None = Field( + default=None, + description="Required headers for inference requests. Cannot be overridden by user requests.", + ) + model_deployment_id: str | None = Field( + default=None, + description="Optional reference to the ModelDeployment ID if this provider was auto-created for a deployment", + max_length=_MAX_LEN_255, + ) + auth_context: AuthContext | None = Field(default=None, description="Auth context captured at provider creation.") + auth_header_format: str | None = Field( + default=None, + description=_AUTH_HEADER_FORMAT_DESCRIPTION, + max_length=1024, + ) + + +class ModelProviderSort(StrEnum): + """Sort fields for ModelProvider queries.""" + + NAME_ASC = "name" + NAME_DESC = "-name" + CREATED_AT_ASC = "created_at" + CREATED_AT_DESC = "-created_at" + UPDATED_AT_ASC = "updated_at" + UPDATED_AT_DESC = "-updated_at" + STATUS_ASC = "status" + STATUS_DESC = "-status" + + +class CreateModelProviderRequest(BaseModel): + """Request model for creating a ModelProvider.""" + + name: str = Field( + description=f"Name of the model provider. {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + examples=["my-nim-provider", "openai-endpoint"], + ) + project: str | None = Field( + default=None, + description="The URN of the project associated with this model provider", + max_length=_MAX_LEN_255, + pattern=_NAME_SLASH_REGEX, + ) + description: str | None = Field( + default=None, + description="Optional description of the model provider", + max_length=1000, + ) + host_url: str = Field( + description="The network endpoint URL for the model provider", + max_length=2048, + ) + api_key_secret_name: str | None = Field( + default=None, + description="Reference to an API key secret stored in the Secrets service. " + "Create the secret first via secrets API, then pass the secret name here.", + max_length=_MAX_LEN_255, + ) + enabled_models: list[str] | None = Field( + default=None, description="Optional list of specific models to enable from this provider" + ) + default_extra_body: dict[str, Any] | None = Field( + default=None, + description="Default body parameters for inference requests. Can be overridden by user requests.", + ) + default_extra_headers: dict[str, str] | None = Field( + default=None, + description="Default headers for inference requests. Can be overridden by user requests.", + ) + required_extra_body: dict[str, Any] | None = Field( + default=None, + description="Required body parameters for inference requests. Cannot be overridden by user requests.", + ) + required_extra_headers: dict[str, str] | None = Field( + default=None, + description="Required headers for inference requests. Cannot be overridden by user requests.", + ) + model_deployment_id: str | None = Field( + default=None, + description="Optional reference to the ModelDeployment ID if this provider is being auto-created for a deployment", + max_length=_MAX_LEN_255, + ) + status: ModelProviderStatus | None = Field(default=None, description="Status of the model provider") + status_message: str | None = Field( + default=None, + description="Status message", + max_length=1000, + ) + auth_header_format: str | None = Field( + default=None, + description=_AUTH_HEADER_FORMAT_DESCRIPTION, + max_length=1024, + ) + + +class UpsertModelProviderRequest(BaseModel): + """Request model for upserting a ModelProvider (PUT). + + All fields must be provided - partial updates are not supported for security reasons. + Use PUT /status endpoint to update status-related fields only. + """ + + project: str | None = Field( + default=None, + description="The URN of the project associated with this model provider", + max_length=_MAX_LEN_255, + pattern=_NAME_SLASH_REGEX, + ) + description: str | None = Field( + default=None, + description="Optional description of the model provider", + max_length=1000, + ) + host_url: str = Field( + description="The network endpoint URL for the model provider", + max_length=2048, + ) + api_key_secret_name: str | None = Field( + default=None, + description="Reference to an API key secret stored in the Secrets service. " + "Create the secret first via secrets API, then pass the secret name here.", + max_length=_MAX_LEN_255, + ) + enabled_models: list[str] | None = Field( + default=None, description="Optional list of specific models to enable from this provider" + ) + default_extra_body: dict[str, Any] | None = Field( + default=None, + description="Default body parameters for inference requests. Can be overridden by user requests.", + ) + default_extra_headers: dict[str, str] | None = Field( + default=None, + description="Default headers for inference requests. Can be overridden by user requests.", + ) + required_extra_body: dict[str, Any] | None = Field( + default=None, + description="Required body parameters for inference requests. Cannot be overridden by user requests.", + ) + required_extra_headers: dict[str, str] | None = Field( + default=None, + description="Required headers for inference requests. Cannot be overridden by user requests.", + ) + model_deployment_id: str | None = Field( + default=None, + description="Optional reference to the ModelDeployment ID if this provider is associated with a deployment", + max_length=_MAX_LEN_255, + ) + status: ModelProviderStatus | None = Field(default=None, description="Status of the model provider") + status_message: str | None = Field( + default=None, + description="Status message", + max_length=1000, + ) + auth_header_format: str | None = Field( + default=None, + description=_AUTH_HEADER_FORMAT_DESCRIPTION, + max_length=1024, + ) + + +class UpdateModelProviderStatusRequest(BaseModel): + """Request model for updating ModelProvider status and autodiscovery fields.""" + + model_deployment_id: str | None = Field( + default=None, + description="Reference to the ModelDeployment ID if this provider is associated with a deployment", + max_length=_MAX_LEN_255, + ) + served_models: list[ServedModelMapping] | None = Field( + default=None, description="List of models served by this provider with routing information for IGW" + ) + status: ModelProviderStatus | None = Field(default=None, description="Status of the model provider") + status_message: str | None = Field( + default=None, + description="Status message. If status is provided without status_message, defaults to empty string.", + max_length=1000, + ) + + +# --------------------------------------------------------------------------- +# Prompt +# --------------------------------------------------------------------------- + + +class PromptMessageRole(StrEnum): + """Role of a message author in a chat prompt.""" + + SYSTEM = "system" + DEVELOPER = "developer" + USER = "user" + ASSISTANT = "assistant" + + +class PromptMessage(BaseModel): + """A single templated message in a chat prompt.""" + + role: PromptMessageRole = Field(description="The role of the message author.") + content: str = Field(description="Templated message content. May contain template variables.") + + +class FunctionDefinition(BaseModel): + """An OpenAI-compatible function definition for tool calling.""" + + name: str = Field( + description="The name of the function to be called.", + max_length=_MAX_LEN_255, + ) + description: str | None = Field( + default=None, + description="A description of what the function does, used by the model to decide when and how to call it.", + ) + parameters: dict[str, Any] | None = Field( + default=None, + description="The parameters the function accepts, described as a JSON Schema object.", + ) + strict: bool | None = Field( + default=None, + description="Whether to enforce strict schema adherence when generating the function call.", + ) + + +class ChatCompletionTool(BaseModel): + """An OpenAI-compatible tool definition (currently always a function tool).""" + + type: str = Field(description="The type of the tool. Currently only 'function' is supported.") + function: FunctionDefinition = Field(description="The function definition for this tool.") + + @field_validator("type") + @classmethod + def _validate_type(cls, v: str) -> str: + if v != "function": + raise ValueError("Only 'function' tools are supported") + return v + + +class Prompt(ModelEntityBaseModel): + """A reusable, stored chat prompt. The unique identifier is workspace/name.""" + + id: str = Field(default="", description="Unique identifier for the prompt.") + description: str | None = Field( + default=None, + description="Optional description of the prompt.", + max_length=1000, + ) + messages: list[PromptMessage] = Field( + default_factory=list, + description="Ordered list of chat messages that make up the prompt.", + ) + input_variables: list[str] = Field( + default_factory=list, + description="Names of the Jinja2 template variables the prompt expects.", + ) + tools: list[ChatCompletionTool] | None = Field( + default=None, + description="Optional OpenAI-compatible tool definitions to send with the prompt.", + ) + tool_choice: str | dict[str, Any] | None = Field( + default=None, + description="Controls which (if any) tool is called: 'none', 'auto', 'required', or a named-tool object.", + ) + response_format: dict[str, Any] | None = Field( + default=None, + description="Optional OpenAI-compatible response_format, e.g. a json_schema structured-output spec.", + ) + inference_params: InferenceParams | None = Field( + default=None, + description="Optional default model and sampling parameters (temperature, top_p, max_tokens, ...).", + ) + tags: list[str] = Field( + default_factory=list, + description="Optional free-form tags for organizing prompts.", + ) + + +class PromptSort(StrEnum): + """Sort fields for Prompt queries.""" + + NAME_ASC = "name" + NAME_DESC = "-name" + CREATED_AT_ASC = "created_at" + CREATED_AT_DESC = "-created_at" + UPDATED_AT_ASC = "updated_at" + UPDATED_AT_DESC = "-updated_at" + + +class CreatePromptRequest(BaseModel): + """Request model for creating a Prompt.""" + + name: str = Field( + description=f"Name of the prompt. {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + examples=["support-bot-system", "summarizer"], + ) + project: str | None = Field( + default=None, + description="The URN of the project associated with this prompt.", + max_length=_MAX_LEN_255, + pattern=_NAME_SLASH_REGEX, + ) + description: str | None = Field(default=None, max_length=1000) + messages: list[PromptMessage] = Field(default_factory=list) + input_variables: list[str] = Field(default_factory=list) + tools: list[ChatCompletionTool] | None = Field(default=None) + tool_choice: str | dict[str, Any] | None = Field(default=None) + response_format: dict[str, Any] | None = Field(default=None) + inference_params: InferenceParams | None = Field(default=None) + tags: list[str] | None = Field(default=None) + + +class UpdatePromptRequest(BaseModel): + """Request model for replacing a Prompt's mutable fields (full update). + + The prompt name and workspace come from the URL path and cannot be changed. + """ + + project: str | None = Field( + default=None, + description="The URN of the project associated with this prompt.", + max_length=_MAX_LEN_255, + pattern=_NAME_SLASH_REGEX, + ) + description: str | None = Field(default=None, max_length=1000) + messages: list[PromptMessage] = Field(default_factory=list) + input_variables: list[str] = Field(default_factory=list) + tools: list[ChatCompletionTool] | None = Field(default=None) + tool_choice: str | dict[str, Any] | None = Field(default=None) + response_format: dict[str, Any] | None = Field(default=None) + inference_params: InferenceParams | None = Field(default=None) + tags: list[str] | None = Field(default=None) + + +# --------------------------------------------------------------------------- +# Model entity + Adapter +# --------------------------------------------------------------------------- + + +class Adapter(BaseModel): + name: str = Field( + ..., + description=f"Name of the adapter. Name must be unique in the workspace for all Adapters and match the following regex: {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + examples=["lora-adapter-v1", "my-finetune"], + ) + workspace: str = Field( + ..., + description=f"Workspace of the adapter. {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + ) + description: str | None = Field( + default=None, + description="Optional description of the adapter", + max_length=1000, + ) + fileset: str = Field( + ..., + description="Fileset where the adapter files are stored expected format {workspace}/{fileset_name}", + ) + finetuning_type: FinetuningType = Field(..., description="Type of finetuning (LORA, P_TUNING, etc.)") + enabled: bool = Field( + default=True, + description="Whether to make this adapter available for inference post training", + ) + lora_config: Lora | None = Field(None, description="Lora configuration specifics") + model: str | None = Field( + default=None, + description=f"Parent model entity reference. {MODEL_REF_PATTERN_DESCRIPTION}", + max_length=MODEL_REF_MAX_LEN, + ) + created_at: datetime = Field(default_factory=datetime.now) + updated_at: datetime = Field(default_factory=datetime.now) + + @field_validator("model") + @classmethod + def validate_model(cls, v: str | None) -> str | None: + if v is not None and not is_valid_model_ref(v): + raise ValueError(MODEL_REF_PATTERN_DESCRIPTION) + return v + + +class ModelEntity(ModelEntityBaseModel): + """A versioned model registered within the platform.""" + + project: str | None = Field( + default=None, + description="The URN of the project associated with this model entity.", + max_length=_MAX_LEN_255, + ) + description: str | None = Field( + default=None, + description="Optional description of the model.", + max_length=1000, + ) + spec: ModelSpec | None = Field(default=None, description="Detailed specification for the model") + finetuning_type: FinetuningType | None = Field(None, description="Set for full weight finetuned models") + fileset: str | None = Field( + default=None, + description="A set of checkpoint files, configs, and other auxiliary info associated with this model - expected format {workspace}/{fileset_name}", + ) + trust_remote_code: bool = Field( + default=False, + description="Whether to trust remote code to load this model checkpoint.", + ) + base_model: str | None = Field( + default=None, description="Link to another model which is used as a base for the current model" + ) + api_endpoint: APIEndpointData | None = Field( + default=None, description="Data about the inference endpoint for this model" + ) + backend_format: BackendFormat | None = Field( + default=None, + description=( + "Inference API wire format expected by the backend. If unset, inference routing treats the model as " + "OPENAI_CHAT." + ), + json_schema_extra={"nullable": True}, + ) + adapters: list[Adapter] | None = Field( + default=None, + description="Adapters that have been created against this model", + ) + prompt: PromptData | None = Field(default=None, description="Configuration for prompt engineering") + custom_fields: dict[str, Any] = Field(default_factory=dict, description="Custom fields for additional metadata") + ownership: dict[str, Any] | None = Field(default=None, description="Ownership information for the model") + model_providers: list[str] = Field( + default_factory=list, + description="List of ModelProvider workspace/name resource names that provide inference for this Model Entity", + ) + + +class CreateModelEntityRequest(BaseModel): + """Request model for creating a Model Entity.""" + + name: str = Field( + description=f"Name of the model entity. {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + examples=["llama-3.1-8b", "my-custom-model"], + ) + project: str | None = Field( + default=None, + description="The URN of the project associated with this model entity", + max_length=_MAX_LEN_255, + pattern=_NAME_SLASH_REGEX, + ) + description: str | None = Field( + default=None, + description="Optional description of the model", + max_length=1000, + ) + spec: ModelSpec | None = Field( + default=None, + description="Detailed specification for the model - Automatically generated by the platform at creation when fileset provided.", + ) + finetuning_type: FinetuningType | None = Field(None, description="Set for full weight finetuned models") + fileset: str | None = Field( + default=None, + description="A set of checkpoint files, configs, and other auxiliary info associated with this model - expected format {workspace}/{fileset_name}", + ) + base_model: str | None = Field( + default=None, description="Link to another model which is used as a base for the current model" + ) + api_endpoint: APIEndpointData | None = Field( + default=None, description="Data about the inference endpoint for this model" + ) + backend_format: BackendFormat | None = Field( + default=None, + description=( + "Inference API wire format expected by the backend. If unset, inference routing treats the model as " + "OPENAI_CHAT." + ), + json_schema_extra={"nullable": True}, + ) + prompt: PromptData | None = Field(default=None, description="Configuration for prompt engineering") + custom_fields: dict[str, Any] | None = Field(default=None, description="Custom fields for additional metadata") + ownership: dict[str, Any] | None = Field(default=None, description="Ownership information for the model") + model_providers: list[str] | None = Field( + default_factory=list, + description="List of ModelProvider workspace/name resource names that provide inference for this Model Entity", + ) + trust_remote_code: bool = Field( + default=False, + description="Whether to trust remote code for the checkpoint.", + ) + + +class CreateModelAdapterRequest(BaseModel): + """Request body for nested Adapter creation. The base model comes from the URL path, not the body.""" + + name: str = Field( + ..., + description=f"Name of the adapter. Name must be unique in the workspace. {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + examples=["lora-adapter-v1", "my-finetune"], + ) + description: str | None = Field( + default=None, + description="Optional description of the adapter", + max_length=1000, + ) + fileset: str = Field( + ..., + description="Location where adapter files are stored - expected format {workspace}/{fileset_name}", + ) + finetuning_type: FinetuningType = Field(..., description="Type of finetuning (LORA, P_TUNING, etc.)") + enabled: bool = Field( + default=True, + description="Whether to make this adapter available for inference post training", + ) + lora_config: Lora | None = Field(None, description="Lora configuration specifics") + + +class CreateAdapterRequest(CreateModelAdapterRequest): + """Request body for Adapter creation.""" + + model: str = Field( + ..., + max_length=MODEL_REF_MAX_LEN, + description=( + f"Base model entity. Use `{{workspace}}/{{model_name}}` to reference a model in any workspace, " + f"or a single `{{model_name}}` resolved in the path workspace. {MODEL_REF_PATTERN_DESCRIPTION}" + ), + examples=["llama-3-8b-instruct", "shared-tenant/base-llm"], + ) + + @field_validator("model") + @classmethod + def validate_model(cls, v: str | None) -> str | None: + if v is not None and not is_valid_model_ref(v): + raise ValueError(MODEL_REF_PATTERN_DESCRIPTION) + return v + + +class UpdateModelEntityRequest(BaseModel): + """Request model for updating Model Entity metadata.""" + + description: str | None = Field( + default=None, + description="Optional description of the model", + max_length=1000, + ) + spec: ModelSpec | None = Field(default=None, description="Detailed specification for the model") + fileset: str | None = Field( + default=None, + description="A set of checkpoint files, configs, and other auxiliary info associated with this model - expected format {workspace}/{fileset_name}", + ) + finetuning_type: FinetuningType | None = Field(None, description="Set for full weight finetuned models") + base_model: str | None = Field( + default=None, description="Link to another model which is used as a base for the current model" + ) + api_endpoint: APIEndpointData | None = Field( + default=None, description="Data about the inference endpoint for this model" + ) + backend_format: BackendFormat | None = Field( + default=None, + description=( + "Inference API wire format expected by the backend. If unset, inference routing treats the model as " + "OPENAI_CHAT." + ), + json_schema_extra={"nullable": True}, + ) + prompt: PromptData | None = Field(default=None, description="Configuration for prompt engineering") + custom_fields: dict[str, Any] | None = Field(default=None, description="Custom fields for additional metadata") + ownership: dict[str, Any] | None = Field(default=None, description="Ownership information for the model") + model_providers: list[str] | None = Field( + default=None, + description="List of ModelProvider workspace/name resource names that provide inference for this Model Entity", + ) + trust_remote_code: bool | None = Field( + default=None, + description="Whether to trust remote code for the checkpoint.", + ) + + +class UpdateAdapterRequest(BaseModel): + """Request model for updating Adapter Sub Entity metadata.""" + + description: str | None = Field( + default=None, + description="Optional description of the adapter", + max_length=1000, + ) + enabled: bool | None = Field( + default=None, + description="Whether to make this adapter available for inference post training", + ) + fileset: str | None = Field( + default=None, + description="Updated fileset for the adapter", + ) + + +class ModelEntitySortField(StrEnum): + """Sort fields for Model Entity queries.""" + + NAME_ASC = "name" + NAME_DESC = "-name" + CREATED_AT_ASC = "created_at" + CREATED_AT_DESC = "-created_at" + UPDATED_AT_ASC = "updated_at" + UPDATED_AT_DESC = "-updated_at" + + +# --------------------------------------------------------------------------- +# ModelDeploymentConfig + ModelDeployment +# --------------------------------------------------------------------------- + + +class ModelType(str, Enum): + """Model type enum for NIM deployments.""" + + LLM = "llm" + EMBED = "embed" + OTHER = "other" + + +class K8sNIMOperatorConfig(BaseModel): + """Kubernetes configuration for NIM deployment via k8s-nim-operator.""" + + resources: dict[str, Any] | None = Field( + default=None, + description="Kubernetes resource requirements including requests and limits. " + "Example: {'requests': {'cpu': '2', 'memory': '8Gi'}, 'limits': {'memory': '16Gi'}}", + ) + tolerations: list[dict[str, Any]] | None = Field( + default=None, + description="Kubernetes tolerations for pod scheduling. " + "Example: [{'key': 'nvidia.com/gpu', 'operator': 'Exists', 'effect': 'NoSchedule'}]", + ) + node_selector: dict[str, str] | None = Field( + default=None, + description="Kubernetes node selector for pod placement. " + "Example: {'node-type': 'gpu-node', 'zone': 'us-west1-a'}", + ) + startup_probe_grace_seconds: int | None = Field( + default=None, + description="Grace period in seconds for NIM startup. " + "Determines how long Kubernetes will wait for the NIM to become ready before restarting it. " + "Example: 600 (10 minutes). " + "Must be a positive integer.", + gt=0, + ) + + +class Engine(str, Enum): + """Inference engine selecting the compiler path for a deployment.""" + + NIM = "nim" + VLLM = "vllm" + GENERIC = "generic" + + +class ModelDeploymentConfigModelSpec(BaseModel): + """What model to serve and how -- independent of the executor it runs on.""" + + model_type: ModelType | None = Field(default=None, description="Type of model being deployed") + model_namespace: str | None = Field( + default=None, + description="Model repository namespace - organization/user namespace as it exists in repo_id.", + max_length=_MAX_LEN_255, + ) + model_name: str | None = Field( + default=None, + description="Model name - model repository name for model weights.", + max_length=_MAX_LEN_255, + ) + model_revision: str | None = Field( + default=None, + description="Model revision (branch, tag, or commit). If not specified, parsed from model_name @revision suffix or defaults to 'main'", + max_length=_MAX_LEN_255, + ) + chat_template: str | None = Field( + default=None, + description="Jinja2 chat template string for the model. Overrides the chat_template from ModelEntity.spec " + "if both are set. Used by the engine to format chat completions.", + ) + tool_call_config: ToolCallConfig | None = Field( + default=None, + description="Tool calling configuration for the deployment. Overrides tool_call_config from " + "ModelEntity.spec if both are set. Controls how the model handles function/tool calling.", + ) + lora_enabled: bool = Field(default=False, description="Whether to enable LoRA support") + + +class ContainerExecutorConfig(BaseModel): + """Compute + container settings shared by the docker and k8s executors.""" + + gpu: int = Field(description="Number of GPUs required for the deployment. 0 = CPU-only.", ge=0) + disk_size: str = Field(default="50Gi", description="Disk size for the deployment") + image_name: str | None = Field( + default=None, + description="Container image name. If not specified, defaults to the engine's configured image " + "(e.g. default_vllm_image / default_nimservice_image). Required for engine='generic'.", + max_length=_MAX_LEN_255, + ) + image_tag: str | None = Field( + default=None, + description="Container image tag. If not specified, defaults to the engine's configured image tag.", + max_length=_MAX_LEN_255, + ) + health_check_path: str | None = Field( + default=None, + description="HTTP path used for the container readiness probe. If not specified, defaults to the " + "engine's standard health endpoint (e.g. '/v1/health/ready' for NIM, '/health' for vLLM). " + "Set this for engine='generic' containers that expose a non-standard health endpoint.", + max_length=_MAX_LEN_255, + ) + run_as_user: int | None = Field( + default=None, + ge=0, + description="Pod securityContext runAsUser (uid) for the serving container (k8s backend only). " + "If unset, the engine default applies (vLLM pins its image's user; generic uses the image's " + "own user). Ignored by the docker backend.", + ) + run_as_group: int | None = Field( + default=None, + ge=0, + description="Pod securityContext runAsGroup (gid) for the serving container (k8s backend only). " + "If unset, the engine default applies. Ignored by the docker backend.", + ) + additional_envs: dict[str, str] | None = Field( + default=None, description="Additional environment variables for the deployment" + ) + additional_args: list[str] = Field( + default_factory=list, + description="Raw container/`serve` args appended verbatim to the container's arg vector.", + ) + k8s_nim_operator_config: K8sNIMOperatorConfig | None = Field( + default=None, + description="Typed Kubernetes configuration for common NIMService Spec fields (NIM engine on k8s). " + "Applied after defaults but before override_config. Ignored by non-NIM engines.", + ) + override_config: dict[str, Any] | None = Field( + default=None, + description="Raw NIMService spec configuration that takes precedence over generated config (NIM engine " + "on k8s). Allows advanced configuration options directly. Ignored by non-NIM engines.", + ) + + +class ModelDeploymentStatus(str, Enum): + """Status enum for ModelDeployment objects.""" + + UNKNOWN = "UNKNOWN" # Terminal + CREATED = "CREATED" + PENDING = "PENDING" + READY = "READY" + ERROR = "ERROR" # Terminal + DELETING = "DELETING" + DELETED = "DELETED" # Terminal + LOST = "LOST" # Terminal + + +class ModelDeploymentStatusHistoryItem(BaseModel): + """Record of a status change in ModelDeployment history.""" + + timestamp: datetime = Field(description="When this status was recorded") + status: ModelDeploymentStatus = Field(description="The status at this point in time") + status_message: str = Field(default="", description="Status message", max_length=1000) + + +class ModelDeploymentConfig(ModelEntityBaseModel): + """Immutable, automatically-versioned deployment config. + + The unique identifier is the combination of workspace/name/entity_version. + """ + + id: str = Field(default="", description="Unique identifier for the deployment config") + entity_version: int = Field(description="Version of this deployment config. Automatically managed.") + description: str | None = Field( + default=None, + description="Optional description of the deployment configuration", + max_length=1000, + ) + engine: Engine = Field(description="Inference engine selecting the compiler path (nim/vllm/generic)") + model_spec: ModelDeploymentConfigModelSpec = Field( + description="What model to serve and how -- independent of the executor it runs on" + ) + executor_config: ContainerExecutorConfig = Field( + description="Compute + container settings for the executor the deployment runs on" + ) + model_entity_id: str | None = Field( + default=None, + description="Optional reference to the base model entity ID for this deployment", + max_length=_MAX_LEN_255, + ) + + +class ModelDeployment(ModelEntityBaseModel): + """A deployed instance of a model with a specific configuration. + + The unique identifier is the combination of workspace/name/entity_version. + """ + + id: str = Field(default="", description="Unique identifier for the deployment") + entity_version: int = Field(description="Version of this deployment. Automatically managed.") + config: str = Field( + description="Reference to the ModelDeploymentConfig name", + max_length=_MAX_LEN_255, + ) + config_version: int = Field(description="Reference to the specific ModelDeploymentConfig version") + status: ModelDeploymentStatus = Field( + default=ModelDeploymentStatus.UNKNOWN, + description="Current status of the deployment, populated by models controller", + ) + status_message: str = Field( + default="", + description="Detailed status message, populated by models controller", + max_length=1000, + ) + status_history: list[ModelDeploymentStatusHistoryItem] = Field( + default_factory=list, + description="History of status changes, ordered chronologically (oldest first)", + ) + model_provider_id: str | None = Field( + default=None, + description="Optional reference to the auto-created ModelProvider workspace/name (format: workspace/name)", + max_length=_MAX_LEN_255, + ) + auth_context: AuthContext | None = Field(default=None, description="Auth context captured at deployment creation. ") + + +class CreateModelDeploymentConfigRequest(BaseModel): + """Request model for creating a ModelDeploymentConfig.""" + + name: str = Field( + description=f"Name of the deployment configuration. {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + examples=["nim-config-v1", "production-config"], + ) + project: str | None = Field( + default=None, + description="The URN of the project associated with this deployment configuration", + max_length=_MAX_LEN_255, + pattern=_NAME_SLASH_REGEX, + ) + description: str | None = Field( + default=None, + description="Optional description of the deployment configuration", + max_length=1000, + ) + engine: Engine = Field(description="Inference engine selecting the compiler path (nim/vllm/generic)") + model_spec: ModelDeploymentConfigModelSpec = Field( + description="What model to serve and how -- independent of the executor it runs on" + ) + executor_config: ContainerExecutorConfig = Field( + description="Compute + container settings for the executor the deployment runs on" + ) + model_entity_id: str | None = Field( + default=None, + description="Optional reference to the base model entity ID for this deployment", + max_length=_MAX_LEN_255, + ) + + +class UpdateModelDeploymentConfigRequest(BaseModel): + """Request model for updating a ModelDeploymentConfig (creates new version).""" + + description: str | None = Field( + default=None, + description="Optional description of the deployment configuration", + max_length=1000, + ) + engine: Engine = Field(description="Inference engine selecting the compiler path (nim/vllm/generic)") + model_spec: ModelDeploymentConfigModelSpec = Field( + description="What model to serve and how -- independent of the executor it runs on" + ) + executor_config: ContainerExecutorConfig = Field( + description="Compute + container settings for the executor the deployment runs on" + ) + model_entity_id: str | None = Field( + default=None, + description="Optional reference to the base model entity ID for this deployment", + max_length=_MAX_LEN_255, + ) + + +class CreateModelDeploymentRequest(BaseModel): + """Request model for creating a ModelDeployment.""" + + name: str = Field( + description=f"Name of the deployment. {_NAME_DESC}", + max_length=_MAX_LEN_255, + pattern=_NAME_REGEX, + examples=["llama-deploy-v1", "production-nim"], + ) + project: str | None = Field( + default=None, + description="The URN of the project associated with this deployment", + max_length=_MAX_LEN_255, + pattern=_NAME_SLASH_REGEX, + ) + config: str = Field( + description="Reference to the ModelDeploymentConfig name", + max_length=_MAX_LEN_255, + ) + config_version: int | None = Field( + default=None, + description="Reference to a specific ModelDeploymentConfig version. If not specified, uses latest.", + ) + + +class UpdateModelDeploymentRequest(BaseModel): + """Request model for updating a ModelDeployment (creates new version).""" + + config: str = Field( + description="Reference to the ModelDeploymentConfig name", + max_length=_MAX_LEN_255, + ) + config_version: int | None = Field( + default=None, + description="Reference to a specific ModelDeploymentConfig version. If not specified, uses latest.", + ) + + +class UpdateModelDeploymentStatusRequest(BaseModel): + """Request model for updating ModelDeployment status.""" + + status: ModelDeploymentStatus = Field(description="New status for the deployment") + status_message: str = Field(default="", description="Detailed status message", max_length=1000) + model_provider_id: str | None = Field( + default=None, + description="Optional reference to the auto-created ModelProvider workspace/name (format: workspace/name)", + max_length=_MAX_LEN_255, + ) + + +# --------------------------------------------------------------------------- +# Query parameter types +# --------------------------------------------------------------------------- + + +class ListModelsQueryParams(TypedDict, total=False): + page: NotRequired[int] + page_size: NotRequired[int] + sort: NotRequired[str] + filter: NotRequired[str] + verbose: NotRequired[bool] + + +class GetModelQueryParams(TypedDict, total=False): + verbose: NotRequired[bool] + + +class ListAdaptersQueryParams(TypedDict, total=False): + page: NotRequired[int] + page_size: NotRequired[int] + sort: NotRequired[str] + filter: NotRequired[str] + + +class ListProvidersQueryParams(TypedDict, total=False): + page: NotRequired[int] + page_size: NotRequired[int] + sort: NotRequired[str] + filter: NotRequired[str] + + +class ListPromptsQueryParams(TypedDict, total=False): + page: NotRequired[int] + page_size: NotRequired[int] + sort: NotRequired[str] + filter: NotRequired[str] + + +class ListDeploymentsQueryParams(TypedDict, total=False): + page: NotRequired[int] + page_size: NotRequired[int] + sort: NotRequired[str] + all_versions: NotRequired[bool] + filter: NotRequired[str] + + +class ListDeploymentConfigsQueryParams(TypedDict, total=False): + page: NotRequired[int] + page_size: NotRequired[int] + sort: NotRequired[str] + filter: NotRequired[str] + + +class UpdateDeploymentStatusQueryParams(TypedDict, total=False): + version: NotRequired[str] diff --git a/packages/nemo_platform_plugin/tests/client/test_method.py b/packages/nemo_platform_plugin/tests/client/test_method.py new file mode 100644 index 0000000000..b54ed711d6 --- /dev/null +++ b/packages/nemo_platform_plugin/tests/client/test_method.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the ``method()`` descriptor that binds endpoints onto client classes. + +Instance-level dispatch is exercised throughout the client and Models suites. What +is pinned here is *class*-level access, which nothing else touches and which every +introspection tool performs: ``Mock(spec=SomeClient)``, ``help()``, ``pydoc``, +autodoc, and anything walking ``dir()``. +""" + +from __future__ import annotations + +import inspect +import pydoc +from unittest.mock import MagicMock, create_autospec + +import pytest +from nemo_platform_plugin.client.method import EndpointMethod +from nemo_platform_plugin.models.client import AsyncModelsClient, ModelsClient + +BASE = "http://test:8000" + + +def _descriptor(name: str) -> EndpointMethod: + return inspect.getattr_static(AsyncModelsClient, name) + + +# --------------------------------------------------------------------------- +# Class-level access +# --------------------------------------------------------------------------- + + +def test_class_level_access_returns_the_descriptor() -> None: + """``__get__`` with no instance hands back the descriptor, per the protocol. + + It used to assert ``obj is not None``, so every one of these raised. + """ + assert isinstance(ModelsClient.create_model, EndpointMethod) + assert isinstance(AsyncModelsClient.create_model, EndpointMethod) + assert ModelsClient.create_model is inspect.getattr_static(ModelsClient, "create_model") + + +def test_mock_spec_against_a_client_class_builds() -> None: + """``Mock(spec=...)`` reads every attribute off the class to classify it. + + This is the failure that surfaced the bug: the models controller test suite + could not spec a mock against its own client. + """ + mock = MagicMock(spec=AsyncModelsClient) + + assert callable(mock.create_model) + with pytest.raises(AttributeError): + mock.create_modle # noqa: B018 a typo must not be silently mockable + + +def test_pydoc_lists_endpoints_with_their_docstrings() -> None: + """Endpoint docs reach help() through the copied ``__doc__``. + + pydoc swallows per-member errors, so this does not discriminate the + class-access fix; what it pins is the attribute copying. + """ + # plain() strips pydoc's backspace-overstrike bolding. + rendered = pydoc.plain(pydoc.render_doc(ModelsClient)) + + assert "delete_deployment" in rendered + assert "Delete a deployment" in rendered + + +def test_create_autospec_does_not_raise() -> None: + """autospec also walks the class. + + The resulting endpoint stubs are *not* callable, because the descriptor is not. + That is a real limitation of this design and is pinned here so it is a + deliberate trade-off rather than a surprise: use ``spec=`` for client mocks. + """ + auto = create_autospec(AsyncModelsClient) + + assert not callable(auto.create_model) + + +# --------------------------------------------------------------------------- +# Identity carried from the endpoint +# --------------------------------------------------------------------------- + + +def test_descriptor_carries_endpoint_identity() -> None: + descriptor = _descriptor("delete_deployment") + + assert descriptor.__name__ == "delete_deployment" + assert descriptor.__doc__ == descriptor.endpoint.__doc__ + assert descriptor.__doc__ # the endpoint really does carry one + assert descriptor.__wrapped__ is descriptor.endpoint + + +def test_signature_is_reachable_by_unwrapping_at_class_level() -> None: + """``inspect.signature`` rejects the descriptor; ``unwrap`` gets past it. + + Pinned because the obvious reading of ``__wrapped__`` is that ``signature()`` + follows it. It does not: it refuses a non-callable before ever looking. + """ + # Both calls are typed as taking a callable; passing the descriptor is the + # behaviour under test, hence the suppressions rather than a cast. + with pytest.raises(TypeError): + inspect.signature(AsyncModelsClient.create_model) # ty: ignore[invalid-argument-type] + + signature = inspect.signature(inspect.unwrap(AsyncModelsClient.create_model)) # ty: ignore[invalid-argument-type] + + assert set(signature.parameters) == {"workspace", "body", "exist_ok"} + assert all(p.kind is inspect.Parameter.KEYWORD_ONLY for p in signature.parameters.values()) + + +def test_descriptor_does_not_leak_the_endpoint_abstractmethod_marker() -> None: + """Endpoints are ``@abstractmethod`` stubs; that marker must not ride along. + + ``functools.update_wrapper`` would copy ``__dict__`` and with it + ``__isabstractmethod__``, which would make any ABCMeta-based client class + uninstantiable. The attributes are copied one by one to avoid exactly that. + """ + descriptor = _descriptor("create_model") + + assert getattr(descriptor.endpoint, "__isabstractmethod__", False) is True + assert not getattr(descriptor, "__isabstractmethod__", False) + assert not getattr(AsyncModelsClient, "__abstractmethods__", frozenset()) + ModelsClient(base_url=BASE, workspace="default") # constructs + + +# --------------------------------------------------------------------------- +# Instance-level dispatch (unchanged, but nothing states it outright) +# --------------------------------------------------------------------------- + + +def test_sync_client_binds_a_plain_callable() -> None: + bound = ModelsClient(base_url=BASE, workspace="default").create_model + + assert callable(bound) + assert not inspect.iscoroutinefunction(bound) + + +def test_async_client_binds_a_coroutine_function() -> None: + bound = AsyncModelsClient(base_url=BASE, workspace="default").create_model + + assert inspect.iscoroutinefunction(bound) + + +def test_bound_method_exposes_the_real_signature() -> None: + """Instance access hands back the wrapped function, so signature() works there.""" + signature = inspect.signature(ModelsClient(base_url=BASE, workspace="default").get_model) + + assert set(signature.parameters) == {"workspace", "name", "query_params"} diff --git a/packages/nemo_platform_plugin/tests/models/test_client.py b/packages/nemo_platform_plugin/tests/models/test_client.py new file mode 100644 index 0000000000..fc78574e27 --- /dev/null +++ b/packages/nemo_platform_plugin/tests/models/test_client.py @@ -0,0 +1,405 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for ModelsClient / AsyncModelsClient via mocked httpx transport.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from nemo_platform_plugin.client.errors import ConflictError, NotFoundError +from nemo_platform_plugin.models.client import AsyncModelsClient, ModelsClient +from nemo_platform_plugin.models.types import ( + CreateModelDeploymentRequest, + CreateModelEntityRequest, + CreateModelProviderRequest, + ModelDeployment, + ModelEntity, + ModelProvider, +) + +BASE = "http://test:8000" + + +def _model_json(name: str = "llama", workspace: str = "default", **extra: object) -> dict: + base = { + "id": f"model-{name}", + "name": name, + "workspace": workspace, + "created_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-01T00:00:00Z", + } + base.update(extra) + return base + + +def _provider_json(status: str = "PENDING", name: str = "my-provider", **extra: object) -> dict: + base = { + "id": f"provider-{name}", + "name": name, + "workspace": "default", + "host_url": "https://api.example.com", + "status": status, + "status_message": "", + "created_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-01T00:00:00Z", + } + base.update(extra) + return base + + +def _deployment_json(status: str = "PENDING", history: list | None = None, **extra: object) -> dict: + base = { + "id": "dep-1", + "name": "my-deploy", + "workspace": "default", + "entity_version": 1, + "config": "cfg", + "config_version": 1, + "status": status, + "status_message": "", + "status_history": history if history is not None else [], + "created_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-01T00:00:00Z", + } + base.update(extra) + return base + + +# --------------------------------------------------------------------------- +# Round-trips +# --------------------------------------------------------------------------- + + +def test_create_model_round_trip() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response(201, request=httpx.Request("POST", BASE), json=_model_json()) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + out = client.create_model(body=CreateModelEntityRequest(name="llama")).data() + + assert isinstance(out, ModelEntity) + assert out.name == "llama" + args, kwargs = http.request.call_args + assert args == ("POST", f"{BASE}/apis/models/v2/workspaces/default/models") + assert kwargs["content"] == b'{"name":"llama"}' + + +def test_list_models_paginates_across_pages() -> None: + http = MagicMock(spec=httpx.Client) + page1 = { + "data": [_model_json("a"), _model_json("b")], + "pagination": { + "page": 1, + "page_size": 2, + "current_page_size": 2, + "total_pages": 2, + "total_results": 3, + }, + } + page2 = { + "data": [_model_json("c")], + "pagination": { + "page": 2, + "page_size": 2, + "current_page_size": 1, + "total_pages": 2, + "total_results": 3, + }, + } + http.request.side_effect = [ + httpx.Response(200, request=httpx.Request("GET", BASE), json=page1), + httpx.Response(200, request=httpx.Request("GET", BASE), json=page2), + ] + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + names = [m.name for m in client.list_models().items()] + assert names == ["a", "b", "c"] + + +def test_get_model_not_found_raises() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response(404, request=httpx.Request("GET", BASE), json={"detail": "not found"}) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + with pytest.raises(NotFoundError) as exc: + client.get_model(name="missing") + assert exc.value.status_code == 404 + + +def test_delete_deployment_returns_none_on_202() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response(202, request=httpx.Request("DELETE", BASE)) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + resp = client.delete_deployment(name="d") + assert resp.data() is None + assert resp.http_response.status_code == 202 + + +def test_delete_deployment_returns_none_on_204() -> None: + """Synchronous hard-delete: 204 No Content is success with a ``None`` body.""" + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response(204, request=httpx.Request("DELETE", BASE)) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + resp = client.delete_deployment(name="d") + assert resp.data() is None + # The 202/204 distinction is only observable via the raw status code. + assert resp.http_response.status_code == 204 + + +def test_delete_deployment_version_accepts_202_and_204() -> None: + http = MagicMock(spec=httpx.Client) + http.request.side_effect = [ + httpx.Response(202, request=httpx.Request("DELETE", BASE)), + httpx.Response(204, request=httpx.Request("DELETE", BASE)), + ] + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert client.delete_deployment_version(deployment="d", name="1").data() is None + assert client.delete_deployment_version(deployment="d", name="2").data() is None + + +def test_create_deployment_conflict_without_exist_ok_raises() -> None: + """Default exist_ok=False: a 409 surfaces as ConflictError (no GET replay).""" + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response(409, request=httpx.Request("POST", BASE), json={"detail": "exists"}) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + with pytest.raises(ConflictError) as exc: + client.create_deployment(body=CreateModelDeploymentRequest(name="d", config="cfg")) + assert exc.value.status_code == 409 + # A single POST was made; no conflict-resolving GET replay happened. + assert http.request.call_count == 1 + + +def test_create_provider_exist_ok_resolves_conflict() -> None: + """exist_ok=True: a 409 replays the linked GET and returns the existing entity.""" + http = MagicMock(spec=httpx.Client) + conflict = httpx.Response(409, request=httpx.Request("POST", BASE), json={"detail": "exists"}) + existing = httpx.Response( + 200, + request=httpx.Request("GET", BASE), + json=_model_json("p", host_url="http://x") | {"host_url": "http://x"}, + ) + http.request.side_effect = [conflict, existing] + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + out = client.create_provider(body=CreateModelProviderRequest(name="p", host_url="http://x"), exist_ok=True).data() + + assert isinstance(out, ModelProvider) + assert out.name == "p" + # Second call is the GET replay for the existing provider. + assert http.request.call_args_list[1].args[0] == "GET" + assert http.request.call_args_list[1].args[1].endswith("/providers/p") + + +# --------------------------------------------------------------------------- +# URL builders +# --------------------------------------------------------------------------- + + +def test_openai_route_base_url() -> None: + client = ModelsClient(base_url=BASE + "/", workspace="default") + assert client.get_openai_route_base_url() == f"{BASE}/apis/inference-gateway/v2/workspaces/default/openai/-/v1" + assert ( + client.get_openai_route_base_url(workspace="other") + == f"{BASE}/apis/inference-gateway/v2/workspaces/other/openai/-/v1" + ) + + +def test_openai_route_base_url_missing_workspace_raises() -> None: + client = ModelsClient(base_url=BASE) + with pytest.raises(ValueError, match="Missing workspace"): + client.get_openai_route_base_url() + + +def test_provider_route_appends_v1_conditionally() -> None: + client = ModelsClient(base_url=BASE, workspace="default") + p_openai = ModelProvider.model_validate(_model_json("p", host_url="https://api.openai.com")) + p_nim = ModelProvider.model_validate(_model_json("p", host_url="https://nim.example.com/v1")) + + assert client.get_provider_route_openai_url(p_openai).endswith("/provider/p/-/v1") + assert client.get_provider_route_openai_url(p_nim).endswith("/provider/p/-") + + +def test_model_entity_route_always_v1() -> None: + client = ModelsClient(base_url=BASE, workspace="default") + me = ModelEntity.model_validate(_model_json("m")) + assert client.get_model_entity_route_openai_url(me).endswith("/model/m/-/v1") + + +def test_provider_route_for_deployment_fetches_provider() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response( + 200, + request=httpx.Request("GET", BASE), + json=_model_json("my-provider", host_url="https://api.example.com"), + ) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + deployment = ModelDeployment.model_validate(_deployment_json(model_provider_id="default/my-provider")) + + url = client.get_provider_route_openai_url_for_deployment(deployment) + assert url.endswith("/workspaces/default/provider/my-provider/-/v1") + assert http.request.call_args.args[1].endswith("/providers/my-provider") + + +def test_provider_route_for_deployment_without_provider_id_raises() -> None: + client = ModelsClient(base_url=BASE, workspace="default") + deployment = ModelDeployment.model_validate(_deployment_json(model_provider_id=None)) + with pytest.raises(ValueError, match="no associated model_provider_id"): + client.get_provider_route_openai_url_for_deployment(deployment) + + +# --------------------------------------------------------------------------- +# Deployment polling +# --------------------------------------------------------------------------- + + +def test_wait_for_deployment_status_reaches_ready() -> None: + http = MagicMock(spec=httpx.Client) + http.request.side_effect = [ + httpx.Response(200, request=httpx.Request("GET", BASE), json=_deployment_json("PENDING")), + httpx.Response(200, request=httpx.Request("GET", BASE), json=_deployment_json("READY")), + ] + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert client.wait_for_deployment_status("my-deploy", "READY", poll_interval=0.0) is True + + +def test_wait_for_deployment_status_deleted_on_404() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response(404, request=httpx.Request("GET", BASE), json={"detail": "x"}) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert client.wait_for_deployment_status("my-deploy", "DELETED", poll_interval=0.0) is True + + +def test_wait_for_deployment_status_error_returns_false() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response( + 200, request=httpx.Request("GET", BASE), json=_deployment_json("ERROR", status_message="boom") + ) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert client.wait_for_deployment_status("my-deploy", "READY", poll_interval=0.0) is False + + +def test_wait_for_deployment_status_uses_history_tail() -> None: + history = [ + {"timestamp": "2020-01-01T00:00:01Z", "status": "PENDING", "status_message": ""}, + {"timestamp": "2020-01-01T00:00:05Z", "status": "READY", "status_message": "up"}, + ] + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response( + 200, request=httpx.Request("GET", BASE), json=_deployment_json("PENDING", history=history) + ) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + # Top-level status is PENDING but history tail is READY -> reached. + assert client.wait_for_deployment_status("my-deploy", "READY", poll_interval=0.0) is True + + +def test_wait_for_deployment_status_timeout() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response( + 200, request=httpx.Request("GET", BASE), json=_deployment_json("PENDING") + ) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert client.wait_for_deployment_status("my-deploy", "READY", timeout=0, poll_interval=0.0) is False + + +# --------------------------------------------------------------------------- +# Provider polling +# --------------------------------------------------------------------------- + + +def test_wait_for_provider_status_reaches_ready() -> None: + http = MagicMock(spec=httpx.Client) + http.request.side_effect = [ + httpx.Response(200, request=httpx.Request("GET", BASE), json=_provider_json("PENDING")), + httpx.Response(200, request=httpx.Request("GET", BASE), json=_provider_json("READY")), + ] + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert client.wait_for_provider_status("my-provider", "READY", poll_interval=0.0) is True + + +def test_wait_for_provider_status_error_returns_false() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response( + 200, request=httpx.Request("GET", BASE), json=_provider_json("ERROR", status_message="boom") + ) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert client.wait_for_provider_status("my-provider", "READY", poll_interval=0.0) is False + + +def test_wait_for_provider_status_not_found_returns_false() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response(404, request=httpx.Request("GET", BASE), json={"detail": "x"}) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert client.wait_for_provider_status("my-provider", "READY", poll_interval=0.0) is False + + +def test_wait_for_provider_status_timeout() -> None: + http = MagicMock(spec=httpx.Client) + http.request.return_value = httpx.Response(200, request=httpx.Request("GET", BASE), json=_provider_json("PENDING")) + client = ModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert client.wait_for_provider_status("my-provider", "READY", timeout=0, poll_interval=0.0) is False + + +# --------------------------------------------------------------------------- +# Async +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_async_create_model() -> None: + http = AsyncMock(spec=httpx.AsyncClient) + http.request.return_value = httpx.Response(201, request=httpx.Request("POST", BASE), json=_model_json()) + client = AsyncModelsClient(base_url=BASE, workspace="default", http_client=http) + + out = (await client.create_model(body=CreateModelEntityRequest(name="llama"))).data() + assert out.name == "llama" + + +@pytest.mark.asyncio +async def test_async_wait_for_deployment_status_ready() -> None: + http = AsyncMock(spec=httpx.AsyncClient) + http.request.side_effect = [ + httpx.Response(200, request=httpx.Request("GET", BASE), json=_deployment_json("PENDING")), + httpx.Response(200, request=httpx.Request("GET", BASE), json=_deployment_json("READY")), + ] + client = AsyncModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert await client.wait_for_deployment_status("my-deploy", "READY", poll_interval=0.0) is True + + +@pytest.mark.asyncio +async def test_async_wait_for_provider_status_ready() -> None: + http = AsyncMock(spec=httpx.AsyncClient) + http.request.side_effect = [ + httpx.Response(200, request=httpx.Request("GET", BASE), json=_provider_json("PENDING")), + httpx.Response(200, request=httpx.Request("GET", BASE), json=_provider_json("READY")), + ] + client = AsyncModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert await client.wait_for_provider_status("my-provider", "READY", poll_interval=0.0) is True + + +@pytest.mark.asyncio +async def test_async_wait_for_provider_status_error_returns_false() -> None: + http = AsyncMock(spec=httpx.AsyncClient) + http.request.return_value = httpx.Response( + 200, request=httpx.Request("GET", BASE), json=_provider_json("ERROR", status_message="boom") + ) + client = AsyncModelsClient(base_url=BASE, workspace="default", http_client=http) + + assert await client.wait_for_provider_status("my-provider", "READY", poll_interval=0.0) is False diff --git a/packages/nemo_platform_plugin/tests/models/test_endpoints.py b/packages/nemo_platform_plugin/tests/models/test_endpoints.py new file mode 100644 index 0000000000..b45eb4b6af --- /dev/null +++ b/packages/nemo_platform_plugin/tests/models/test_endpoints.py @@ -0,0 +1,312 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for Models service endpoint definitions (PreparedRequest shape).""" + +from __future__ import annotations + +import json +from typing import get_origin + +from nemo_platform_plugin.client.types import Paginated, PreparedRequest +from nemo_platform_plugin.models import endpoints +from nemo_platform_plugin.models.types import ( + Adapter, + ContainerExecutorConfig, + CreateAdapterRequest, + CreateModelDeploymentConfigRequest, + CreateModelDeploymentRequest, + CreateModelEntityRequest, + CreateModelProviderRequest, + CreatePromptRequest, + Engine, + FinetuningType, + ModelDeployment, + ModelDeploymentConfig, + ModelDeploymentConfigModelSpec, + ModelDeploymentStatus, + ModelEntity, + ModelProvider, + ModelProviderStatus, + Prompt, + UpdateAdapterRequest, + UpdateModelDeploymentRequest, + UpdateModelDeploymentStatusRequest, + UpdateModelEntityRequest, + UpdateModelProviderStatusRequest, + UpdatePromptRequest, + UpsertModelProviderRequest, +) + +_PREFIX = "/apis/models/v2/workspaces/{workspace}" + + +def _json_body(prepared: PreparedRequest) -> dict: + """Decode a prepared request's JSON body (asserting it is present bytes).""" + assert isinstance(prepared.content, bytes) + return json.loads(prepared.content) + + +# --------------------------------------------------------------------------- +# Model entities +# --------------------------------------------------------------------------- + + +def test_create_model() -> None: + prepared = endpoints.create_model(workspace="default", body=CreateModelEntityRequest(name="llama")) + assert isinstance(prepared, PreparedRequest) + assert prepared.method == "POST" + assert prepared.path_template == _PREFIX + "/models" + assert prepared.path_params == {"workspace": "default"} + assert prepared.content_type == "application/json" + assert prepared.response_type is ModelEntity + # exist_ok wiring: conflict resolver prebuilt, but not requested by default. + assert prepared.on_conflict_get is not None + assert prepared.on_conflict_get.method == "GET" + assert prepared.on_conflict_get.path_params == {"workspace": "default", "name": "llama"} + + +def test_create_model_excludes_unset() -> None: + prepared = endpoints.create_model(workspace="w", body=CreateModelEntityRequest(name="m")) + assert _json_body(prepared) == {"name": "m"} + + +def test_create_model_workspace_optional() -> None: + prepared = endpoints.create_model(body=CreateModelEntityRequest(name="m")) + assert prepared.path_params == {} + + +def test_list_models_paginated_with_query() -> None: + prepared = endpoints.list_models( + workspace="default", query_params={"page": 2, "sort": "-created_at", "verbose": True} + ) + assert prepared.method == "GET" + assert prepared.path_template == _PREFIX + "/models" + assert get_origin(prepared.response_type) is Paginated + assert prepared.query_params == {"page": 2, "sort": "-created_at", "verbose": True} + + +def test_get_model_with_verbose() -> None: + prepared = endpoints.get_model(workspace="default", name="m", query_params={"verbose": True}) + assert prepared.method == "GET" + assert prepared.path_params == {"workspace": "default", "name": "m"} + assert prepared.query_params == {"verbose": True} + assert prepared.response_type is ModelEntity + + +def test_update_model_patch_with_verbose() -> None: + prepared = endpoints.update_model( + workspace="default", name="m", body=UpdateModelEntityRequest(description="d"), query_params={"verbose": False} + ) + assert prepared.method == "PATCH" + assert prepared.path_params == {"workspace": "default", "name": "m"} + assert _json_body(prepared) == {"description": "d"} + assert prepared.query_params == {"verbose": False} + + +def test_delete_model_returns_none() -> None: + prepared = endpoints.delete_model(workspace="default", name="m") + assert prepared.method == "DELETE" + assert prepared.content is None + assert prepared.response_type is None + + +# --------------------------------------------------------------------------- +# Adapters (nested + top-level) +# --------------------------------------------------------------------------- + + +def test_create_model_adapter_nested_path() -> None: + prepared = endpoints.create_model_adapter( + workspace="w", + model_name="base", + body=__import__( + "nemo_platform_plugin.models.types", fromlist=["CreateModelAdapterRequest"] + ).CreateModelAdapterRequest(name="a", fileset="w/fs", finetuning_type=FinetuningType.LORA), + ) + assert prepared.method == "POST" + assert prepared.path_template == _PREFIX + "/models/{model_name}/adapters" + assert prepared.path_params == {"workspace": "w", "model_name": "base"} + assert prepared.response_type is Adapter + + +def test_update_model_adapter_path() -> None: + prepared = endpoints.update_model_adapter( + workspace="w", model_name="base", adapter="a", body=UpdateAdapterRequest(enabled=False) + ) + assert prepared.method == "PATCH" + assert prepared.path_params == {"workspace": "w", "model_name": "base", "adapter": "a"} + assert _json_body(prepared) == {"enabled": False} + + +def test_delete_model_adapter_path() -> None: + prepared = endpoints.delete_model_adapter(workspace="w", model_name="base", adapter="a") + assert prepared.method == "DELETE" + assert prepared.path_template == _PREFIX + "/models/{model_name}/adapters/{adapter}" + assert prepared.response_type is None + + +def test_create_adapter_top_level_conflict_resolver() -> None: + body = CreateAdapterRequest(name="a", fileset="w/fs", finetuning_type=FinetuningType.LORA, model="ws/base") + prepared = endpoints.create_adapter(workspace="w", body=body) + assert prepared.path_template == _PREFIX + "/adapters" + assert prepared.response_type is Adapter + assert prepared.on_conflict_get is not None + assert prepared.on_conflict_get.path_params == {"workspace": "w", "name": "a"} + + +def test_list_adapters_paginated() -> None: + prepared = endpoints.list_adapters(workspace="w", query_params={"filter": "name:a"}) + assert get_origin(prepared.response_type) is Paginated + assert prepared.query_params == {"filter": "name:a"} + + +def test_get_and_delete_adapter() -> None: + assert endpoints.get_adapter(workspace="w", name="a").response_type is Adapter + assert endpoints.delete_adapter(workspace="w", name="a").method == "DELETE" + + +# --------------------------------------------------------------------------- +# Model providers +# --------------------------------------------------------------------------- + + +def test_create_provider() -> None: + prepared = endpoints.create_provider(workspace="w", body=CreateModelProviderRequest(name="p", host_url="http://x")) + assert prepared.method == "POST" + assert prepared.path_template == _PREFIX + "/providers" + assert prepared.response_type is ModelProvider + assert prepared.on_conflict_get is not None + + +def test_upsert_provider_is_put() -> None: + prepared = endpoints.upsert_provider(workspace="w", name="p", body=UpsertModelProviderRequest(host_url="http://x")) + assert prepared.method == "PUT" + assert prepared.path_template == _PREFIX + "/providers/{name}" + assert prepared.response_type is ModelProvider + + +def test_update_provider_status_is_put_status_path() -> None: + prepared = endpoints.update_provider_status( + workspace="w", name="p", body=UpdateModelProviderStatusRequest(status=ModelProviderStatus.READY) + ) + assert prepared.method == "PUT" + assert prepared.path_template == _PREFIX + "/providers/{name}/status" + + +def test_list_get_delete_provider() -> None: + assert get_origin(endpoints.list_providers(workspace="w").response_type) is Paginated + assert endpoints.get_provider(workspace="w", name="p").response_type is ModelProvider + assert endpoints.delete_provider(workspace="w", name="p").response_type is None + + +# --------------------------------------------------------------------------- +# Prompts +# --------------------------------------------------------------------------- + + +def test_prompt_crud_paths() -> None: + assert endpoints.create_prompt(workspace="w", body=CreatePromptRequest(name="p")).method == "POST" + assert endpoints.update_prompt(workspace="w", name="p", body=UpdatePromptRequest()).method == "PUT" + assert endpoints.get_prompt(workspace="w", name="p").response_type is Prompt + assert get_origin(endpoints.list_prompts(workspace="w").response_type) is Paginated + assert endpoints.delete_prompt(workspace="w", name="p").response_type is None + + +# --------------------------------------------------------------------------- +# Deployments +# --------------------------------------------------------------------------- + + +def _create_deployment_body() -> CreateModelDeploymentRequest: + return CreateModelDeploymentRequest(name="d", config="cfg") + + +def test_create_deployment() -> None: + prepared = endpoints.create_deployment(workspace="w", body=_create_deployment_body()) + assert prepared.method == "POST" + assert prepared.path_template == _PREFIX + "/deployments" + assert prepared.response_type is ModelDeployment + + +def test_update_deployment_is_post_name_path() -> None: + prepared = endpoints.update_deployment(workspace="w", name="d", body=UpdateModelDeploymentRequest(config="cfg")) + assert prepared.method == "POST" + assert prepared.path_template == _PREFIX + "/deployments/{name}" + + +def test_update_deployment_status_with_version_query() -> None: + prepared = endpoints.update_deployment_status( + workspace="w", + name="d", + body=UpdateModelDeploymentStatusRequest(status=ModelDeploymentStatus.READY), + query_params={"version": "2"}, + ) + assert prepared.method == "POST" + assert prepared.path_template == _PREFIX + "/deployments/{name}/status" + assert prepared.query_params == {"version": "2"} + + +def test_deployment_versions_and_models() -> None: + assert endpoints.list_deployment_versions(workspace="w", name="d").response_type == list[ModelDeployment] + assert endpoints.get_deployment_version(workspace="w", deployment="d", name="2").response_type is ModelDeployment + models_ep = endpoints.get_deployment_models(workspace="w", name="d") + assert models_ep.path_template == _PREFIX + "/deployments/{name}/models" + + +def test_delete_deployment_and_version_return_none() -> None: + assert endpoints.delete_deployment(workspace="w", name="d").response_type is None + assert ( + endpoints.delete_deployment_version(workspace="w", deployment="d", name="2").path_template + == _PREFIX + "/deployments/{deployment}/versions/{name}" + ) + + +# --------------------------------------------------------------------------- +# Deployment configs +# --------------------------------------------------------------------------- + + +def _create_config_body() -> CreateModelDeploymentConfigRequest: + return CreateModelDeploymentConfigRequest( + name="cfg", + engine=Engine.VLLM, + model_spec=ModelDeploymentConfigModelSpec(model_name="llama"), + executor_config=ContainerExecutorConfig(gpu=1), + ) + + +def test_deployment_config_crud_paths() -> None: + create = endpoints.create_deployment_config(workspace="w", body=_create_config_body()) + assert create.method == "POST" + assert create.path_template == _PREFIX + "/deployment-configs" + assert create.response_type is ModelDeploymentConfig + assert create.on_conflict_get is not None + + update = endpoints.update_deployment_config( + workspace="w", + name="cfg", + body=__import__( + "nemo_platform_plugin.models.types", fromlist=["UpdateModelDeploymentConfigRequest"] + ).UpdateModelDeploymentConfigRequest( + engine=Engine.VLLM, + model_spec=ModelDeploymentConfigModelSpec(model_name="llama"), + executor_config=ContainerExecutorConfig(gpu=1), + ), + ) + assert update.method == "POST" + assert update.path_template == _PREFIX + "/deployment-configs/{name}" + + assert ( + endpoints.list_deployment_config_versions(workspace="w", name="cfg").response_type + == list[ModelDeploymentConfig] + ) + assert ( + endpoints.get_deployment_config_version(workspace="w", config="cfg", name="1").response_type + is ModelDeploymentConfig + ) + assert endpoints.delete_deployment_config(workspace="w", name="cfg").response_type is None + assert ( + endpoints.delete_deployment_config_version(workspace="w", config="cfg", name="1").path_template + == _PREFIX + "/deployment-configs/{config}/versions/{name}" + ) From 8103d6ff6f87a14c3cdaf0bf40d3b324ae7347a0 Mon Sep 17 00:00:00 2001 From: Max Dubrinsky Date: Wed, 12 Aug 2026 13:52:11 -0400 Subject: [PATCH 2/4] fix(models): async-mockable client, non-stale delete status, guarded provider id Address review findings on the typed Models client foundation: - method() class-level access returns a per-owning-class callable stub (async def for async clients, def for sync) instead of the raw descriptor, so unittest.mock classifies async endpoints as AsyncMock. Previously Mock(spec=AsyncModelsClient).create_model was a sync MagicMock and could not be awaited, and create_autospec yielded non-callable stubs -- defeating the typed async client's purpose. - delete_deployment appends a DELETING entry to status_history so the client (which reads status_history[-1] as current) no longer sees a stale status after a delete request. - get_provider_route_openai_url_for_deployment guards a model_provider_id that lacks the workspace/ prefix with a clear ValueError instead of an opaque unpack crash. Adds regression tests for all three. Signed-off-by: Max Dubrinsky --- .../src/nemo_platform_plugin/client/method.py | 54 +++++++++++++--- .../src/nemo_platform_plugin/models/client.py | 27 ++++++-- .../tests/client/test_method.py | 63 +++++++++++-------- .../tests/models/test_client.py | 8 +++ .../api/service/model_deployment_service.py | 8 +++ .../test_model_deployment_service_unit.py | 25 ++++++++ 6 files changed, 146 insertions(+), 39 deletions(-) diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/method.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/method.py index 71e14ef12b..cb8c452b5b 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/method.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/client/method.py @@ -82,16 +82,19 @@ class EndpointMethod(Generic[P, SyncReturnT, AsyncReturnT]): def __init__(self, endpoint_fn: Callable[P, PreparedRequest]) -> None: self._endpoint_fn = endpoint_fn + # Per-owning-class callable stubs handed out on *class*-level access, so + # unittest.mock / inspect can classify each endpoint (sync vs coroutine) + # and autospec it as callable. Keyed and cached by objtype so repeated + # class access is stable. See __get__. + self._class_stubs: dict[type | None, Callable[..., object]] = {} # Carry the endpoint's name, docstring, and annotations onto the descriptor # so help() and autodoc describe the endpoint rather than the descriptor. # Set directly rather than via functools.update_wrapper, which expects a # callable wrapper; a descriptor is not one, and which would also copy # __dict__ and with it the endpoint's __isabstractmethod__ marker. # - # This does NOT make inspect.signature(SomeClient.method) work: signature() - # rejects a non-callable before it ever consults __wrapped__. Reach the - # parameter list via inspect.unwrap() at class level, or just read it off - # an instance, where __get__ hands back the bound function. + # inspect.signature(SomeClient.method) works because class-level access + # returns a functools.wraps'd stub (see __get__), not the raw descriptor. self.__wrapped__ = endpoint_fn for attr in functools.WRAPPER_ASSIGNMENTS: try: @@ -99,6 +102,40 @@ def __init__(self, endpoint_fn: Callable[P, PreparedRequest]) -> None: except AttributeError: pass + def _class_level_stub(self, objtype: type | None) -> Callable[..., object]: + """Callable stub returned on class-level access, matched to ``objtype``. + + ``unittest.mock`` (both ``Mock(spec=...)`` and ``create_autospec``) reads + each attribute off the *class* and classifies it with ``callable()`` and + ``asyncio.iscoroutinefunction()``. A bare descriptor is neither callable + nor a coroutine function, so every endpoint on an async client would be + mocked as a sync ``MagicMock`` and could not be awaited. This hands back a + real function -- ``async def`` for async clients, ``def`` for sync -- that + wraps the endpoint (so ``inspect.signature`` works and autospec validates + call signatures), but refuses to run unbound: endpoints only mean anything + against a client instance. + """ + cached = self._class_stubs.get(objtype) + if cached is not None: + return cached + is_async = objtype is not None and issubclass(objtype, AsyncNemoClient) + if is_async: + + @functools.wraps(self._endpoint_fn) + async def stub(*args: object, **kwargs: object) -> object: + raise TypeError(f"{self.__name__} must be called on a client instance, not the class") + else: + + @functools.wraps(self._endpoint_fn) + def stub(*args: object, **kwargs: object) -> object: + raise TypeError(f"{self.__name__} must be called on a client instance, not the class") + + # __isabstractmethod__ rides along in the endpoint's __dict__ via wraps; + # drop it so the stub is never mistaken for an abstract member. + stub.__dict__.pop("__isabstractmethod__", None) + self._class_stubs[objtype] = stub + return stub + @property def endpoint(self) -> Callable[P, PreparedRequest]: """The endpoint function this descriptor binds.""" @@ -114,9 +151,12 @@ def __get__(self, obj: AsyncNemoClient, objtype: type | None = None) -> Callable def __get__(self, obj: NemoClient | AsyncNemoClient | None, objtype: type | None = None) -> object: if obj is None: # Class-level access. Anything that inspects a client class rather than - # an instance -- Mock(spec=...), inspect, help(), autodoc -- lands here, - # and the descriptor protocol says to hand back the descriptor itself. - return self + # an instance -- Mock(spec=...), autospec, inspect, help(), autodoc -- + # lands here. Hand back a callable stub matched to the owning client + # type so mock classifies sync vs async correctly and autospec sees a + # callable. The raw descriptor is still reachable via + # inspect.getattr_static, which never invokes __get__. + return self._class_level_stub(objtype) if isinstance(obj, AsyncNemoClient): @functools.wraps(self._endpoint_fn) diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py index 914f50c43b..232ed45576 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py @@ -82,6 +82,25 @@ def name(self) -> str: ... def model_provider_id(self) -> str | None: ... +def _split_provider_id(deployment: DeploymentLike) -> tuple[str, str]: + """Split a deployment's ``model_provider_id`` into ``(workspace, name)``. + + ``model_provider_id`` is a ``workspace/name`` pair. Raise a clear ValueError + when it is absent or missing the ``workspace/`` prefix, rather than letting an + opaque unpack error surface. + """ + provider_id = deployment.model_provider_id + if not provider_id: + raise ValueError(f"Deployment '{deployment.name}' has no associated model_provider_id") + if "/" not in provider_id: + raise ValueError( + f"Deployment '{deployment.name}' has malformed model_provider_id " + f"'{provider_id}'; expected 'workspace/name'" + ) + workspace, name = provider_id.split("/", 1) + return workspace, name + + def _seconds_since_creation(entry_timestamp: datetime | str | None, created_at: datetime | None) -> int | None: """Seconds from deployment creation to the entry timestamp, or None if not comparable.""" if created_at is None or entry_timestamp is None: @@ -236,9 +255,7 @@ class ModelsClient(_ModelsMethods, _ModelsUrlMixin, NemoClient): def get_provider_route_openai_url_for_deployment(self, deployment: DeploymentLike) -> str: """Fetch a deployment's ModelProvider and return its OpenAI route URL.""" - if not deployment.model_provider_id: - raise ValueError(f"Deployment '{deployment.name}' has no associated model_provider_id") - workspace, name = deployment.model_provider_id.split("/", 1) + workspace, name = _split_provider_id(deployment) provider = self.get_provider(name=name, workspace=workspace).data() return self.get_provider_route_openai_url(provider) @@ -333,9 +350,7 @@ class AsyncModelsClient(_ModelsMethods, _ModelsUrlMixin, AsyncNemoClient): async def get_provider_route_openai_url_for_deployment(self, deployment: DeploymentLike) -> str: """Fetch a deployment's ModelProvider and return its OpenAI route URL.""" - if not deployment.model_provider_id: - raise ValueError(f"Deployment '{deployment.name}' has no associated model_provider_id") - workspace, name = deployment.model_provider_id.split("/", 1) + workspace, name = _split_provider_id(deployment) provider = (await self.get_provider(name=name, workspace=workspace)).data() return self.get_provider_route_openai_url(provider) diff --git a/packages/nemo_platform_plugin/tests/client/test_method.py b/packages/nemo_platform_plugin/tests/client/test_method.py index b54ed711d6..23c05949db 100644 --- a/packages/nemo_platform_plugin/tests/client/test_method.py +++ b/packages/nemo_platform_plugin/tests/client/test_method.py @@ -11,9 +11,10 @@ from __future__ import annotations +import asyncio import inspect import pydoc -from unittest.mock import MagicMock, create_autospec +from unittest.mock import AsyncMock, MagicMock, create_autospec import pytest from nemo_platform_plugin.client.method import EndpointMethod @@ -31,14 +32,27 @@ def _descriptor(name: str) -> EndpointMethod: # --------------------------------------------------------------------------- -def test_class_level_access_returns_the_descriptor() -> None: - """``__get__`` with no instance hands back the descriptor, per the protocol. +def test_class_level_access_returns_a_typed_callable_stub() -> None: + """``__get__`` with no instance hands back a callable stub matched to the class. - It used to assert ``obj is not None``, so every one of these raised. + The stub exists so ``unittest.mock`` / ``inspect`` can classify each endpoint + (sync vs coroutine) and autospec it as callable. The raw descriptor is still + reachable via ``inspect.getattr_static``, which never invokes ``__get__``. """ - assert isinstance(ModelsClient.create_model, EndpointMethod) - assert isinstance(AsyncModelsClient.create_model, EndpointMethod) - assert ModelsClient.create_model is inspect.getattr_static(ModelsClient, "create_model") + assert callable(ModelsClient.create_model) + assert callable(AsyncModelsClient.create_model) + assert not inspect.iscoroutinefunction(ModelsClient.create_model) + assert inspect.iscoroutinefunction(AsyncModelsClient.create_model) + # The descriptor itself is reachable without triggering __get__. + assert isinstance(inspect.getattr_static(ModelsClient, "create_model"), EndpointMethod) + + +def test_class_level_stub_refuses_to_run_unbound() -> None: + """The stub is for introspection only; calling it without an instance errors.""" + with pytest.raises(TypeError): + ModelsClient.create_model(workspace="w", body=None) + with pytest.raises(TypeError): + asyncio.run(AsyncModelsClient.create_model(workspace="w", body=None)) def test_mock_spec_against_a_client_class_builds() -> None: @@ -50,6 +64,9 @@ def test_mock_spec_against_a_client_class_builds() -> None: mock = MagicMock(spec=AsyncModelsClient) assert callable(mock.create_model) + # The async client's endpoints spec as awaitables, not sync MagicMocks -- + # this is the whole point of the typed async client. + assert isinstance(mock.create_model, AsyncMock) with pytest.raises(AttributeError): mock.create_modle # noqa: B018 a typo must not be silently mockable @@ -67,16 +84,19 @@ def test_pydoc_lists_endpoints_with_their_docstrings() -> None: assert "Delete a deployment" in rendered -def test_create_autospec_does_not_raise() -> None: - """autospec also walks the class. +def test_create_autospec_yields_awaitable_endpoint_stubs() -> None: + """autospec walks the class and gets callable, correctly-async stubs. - The resulting endpoint stubs are *not* callable, because the descriptor is not. - That is a real limitation of this design and is pinned here so it is a - deliberate trade-off rather than a surprise: use ``spec=`` for client mocks. + Sync clients autospec to callable MagicMocks; async clients to AsyncMocks, so + ``await auto.create_model(...)`` works and signature validation is enforced. """ - auto = create_autospec(AsyncModelsClient) + auto_sync = create_autospec(ModelsClient) + assert callable(auto_sync.create_model) + assert not isinstance(auto_sync.create_model, AsyncMock) - assert not callable(auto.create_model) + auto_async = create_autospec(AsyncModelsClient) + assert callable(auto_async.create_model) + assert isinstance(auto_async.create_model, AsyncMock) # --------------------------------------------------------------------------- @@ -93,18 +113,9 @@ def test_descriptor_carries_endpoint_identity() -> None: assert descriptor.__wrapped__ is descriptor.endpoint -def test_signature_is_reachable_by_unwrapping_at_class_level() -> None: - """``inspect.signature`` rejects the descriptor; ``unwrap`` gets past it. - - Pinned because the obvious reading of ``__wrapped__`` is that ``signature()`` - follows it. It does not: it refuses a non-callable before ever looking. - """ - # Both calls are typed as taking a callable; passing the descriptor is the - # behaviour under test, hence the suppressions rather than a cast. - with pytest.raises(TypeError): - inspect.signature(AsyncModelsClient.create_model) # ty: ignore[invalid-argument-type] - - signature = inspect.signature(inspect.unwrap(AsyncModelsClient.create_model)) # ty: ignore[invalid-argument-type] +def test_signature_is_reachable_at_class_level() -> None: + """The class-level stub is a real function, so ``signature()`` works directly.""" + signature = inspect.signature(AsyncModelsClient.create_model) assert set(signature.parameters) == {"workspace", "body", "exist_ok"} assert all(p.kind is inspect.Parameter.KEYWORD_ONLY for p in signature.parameters.values()) diff --git a/packages/nemo_platform_plugin/tests/models/test_client.py b/packages/nemo_platform_plugin/tests/models/test_client.py index fc78574e27..245940edcf 100644 --- a/packages/nemo_platform_plugin/tests/models/test_client.py +++ b/packages/nemo_platform_plugin/tests/models/test_client.py @@ -254,6 +254,14 @@ def test_provider_route_for_deployment_without_provider_id_raises() -> None: client.get_provider_route_openai_url_for_deployment(deployment) +def test_provider_route_for_deployment_malformed_id_raises() -> None: + """A provider id without a ``workspace/`` prefix raises a clear error, not an unpack crash.""" + client = ModelsClient(base_url=BASE, workspace="default") + deployment = ModelDeployment.model_validate(_deployment_json(model_provider_id="no-slash-here")) + with pytest.raises(ValueError, match="malformed model_provider_id"): + client.get_provider_route_openai_url_for_deployment(deployment) + + # --------------------------------------------------------------------------- # Deployment polling # --------------------------------------------------------------------------- diff --git a/services/core/models/src/nmp/core/models/api/service/model_deployment_service.py b/services/core/models/src/nmp/core/models/api/service/model_deployment_service.py index bea06b4176..ff15a9d1f7 100644 --- a/services/core/models/src/nmp/core/models/api/service/model_deployment_service.py +++ b/services/core/models/src/nmp/core/models/api/service/model_deployment_service.py @@ -494,6 +494,11 @@ async def delete_deployment(self, workspace: str, name: str, version: int | None # Otherwise, mark for deletion entity.status = ModelDeploymentStatus.DELETING entity.status_message = "Deployment deletion requested" + # Append to history so consumers reading status_history[-1] see the + # DELETING state (the history is the client's source of current status). + entity.status_history.append( + _status_history_entry(datetime.now(timezone.utc), entity.status, entity.status_message) + ) updated = await self.entity_client.update(entity) logger.info(f"Marked deployment for deletion: {workspace}/{name} version {version}") return _entity_to_schema(updated) @@ -530,6 +535,9 @@ async def delete_deployment(self, workspace: str, name: str, version: int | None if entity.status != ModelDeploymentStatus.DELETED: entity.status = ModelDeploymentStatus.DELETING entity.status_message = "Deployment deletion requested" + entity.status_history.append( + _status_history_entry(datetime.now(timezone.utc), entity.status, entity.status_message) + ) updated = await self.entity_client.update(entity) latest_updated = updated diff --git a/services/core/models/tests/unit/test_model_deployment_service_unit.py b/services/core/models/tests/unit/test_model_deployment_service_unit.py index aef88ff491..0993bd6902 100644 --- a/services/core/models/tests/unit/test_model_deployment_service_unit.py +++ b/services/core/models/tests/unit/test_model_deployment_service_unit.py @@ -573,6 +573,31 @@ async def test_delete_deployment_marks_deleting(deployment_service, mock_entity_ mock_entity_client.delete.assert_not_called() +@pytest.mark.asyncio +async def test_delete_deployment_appends_deleting_to_status_history( + deployment_service, mock_entity_client, sample_deployment_entity +): + """Marking a deployment DELETING must append a status_history entry. + + Clients read the current status from ``status_history[-1]``; if the delete + path mutates ``status`` without appending, they see a stale status. + """ + # Arrange + sample_deployment_entity.status_history = [] + mock_entity_client.get.return_value = sample_deployment_entity + mock_entity_client.update.side_effect = lambda entity: entity + + # Act + await deployment_service.delete_deployment("default", "test-deployment", version=1) + + # Assert: the entity handed to update() carries a DELETING history tail. + (updated_entity,), _ = mock_entity_client.update.call_args + assert updated_entity.status_history + last = updated_entity.status_history[-1] + assert last["status"] == ModelDeploymentStatus.DELETING.value + assert last["status_message"] == "Deployment deletion requested" + + @pytest.mark.asyncio async def test_delete_deployment_already_deleted_hard_deletes(deployment_service, mock_entity_client): """Test that deleting a DELETED deployment performs hard delete.""" From 847b28fb2af279fac213032cdea4aaacd162dfe5 Mon Sep 17 00:00:00 2001 From: Max Dubrinsky Date: Wed, 12 Aug 2026 13:55:49 -0400 Subject: [PATCH 3/4] test(models): use top-level imports instead of dynamic __import__ Replace two call-time __import__("...types", fromlist=[...]) lookups in the endpoint tests with normal names added to the existing top-level import block (CreateModelAdapterRequest, UpdateModelDeploymentConfigRequest). Addresses a CodeRabbit maintainability nit on PR #993. Signed-off-by: Max Dubrinsky --- .../tests/models/test_endpoints.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/nemo_platform_plugin/tests/models/test_endpoints.py b/packages/nemo_platform_plugin/tests/models/test_endpoints.py index b45eb4b6af..e1bb55df30 100644 --- a/packages/nemo_platform_plugin/tests/models/test_endpoints.py +++ b/packages/nemo_platform_plugin/tests/models/test_endpoints.py @@ -14,6 +14,7 @@ Adapter, ContainerExecutorConfig, CreateAdapterRequest, + CreateModelAdapterRequest, CreateModelDeploymentConfigRequest, CreateModelDeploymentRequest, CreateModelEntityRequest, @@ -30,6 +31,7 @@ ModelProviderStatus, Prompt, UpdateAdapterRequest, + UpdateModelDeploymentConfigRequest, UpdateModelDeploymentRequest, UpdateModelDeploymentStatusRequest, UpdateModelEntityRequest, @@ -120,9 +122,7 @@ def test_create_model_adapter_nested_path() -> None: prepared = endpoints.create_model_adapter( workspace="w", model_name="base", - body=__import__( - "nemo_platform_plugin.models.types", fromlist=["CreateModelAdapterRequest"] - ).CreateModelAdapterRequest(name="a", fileset="w/fs", finetuning_type=FinetuningType.LORA), + body=CreateModelAdapterRequest(name="a", fileset="w/fs", finetuning_type=FinetuningType.LORA), ) assert prepared.method == "POST" assert prepared.path_template == _PREFIX + "/models/{model_name}/adapters" @@ -286,9 +286,7 @@ def test_deployment_config_crud_paths() -> None: update = endpoints.update_deployment_config( workspace="w", name="cfg", - body=__import__( - "nemo_platform_plugin.models.types", fromlist=["UpdateModelDeploymentConfigRequest"] - ).UpdateModelDeploymentConfigRequest( + body=UpdateModelDeploymentConfigRequest( engine=Engine.VLLM, model_spec=ModelDeploymentConfigModelSpec(model_name="llama"), executor_config=ContainerExecutorConfig(gpu=1), From ee96c89f8bd4f41c884664914a29a20c8065fa61 Mon Sep 17 00:00:00 2001 From: Max Dubrinsky Date: Wed, 12 Aug 2026 14:30:49 -0400 Subject: [PATCH 4/4] fix(models): ruff-format client.py and suppress class-level ty errors CI Lint all caught two issues from the earlier commits: - ruff format collapses the malformed-model_provider_id message onto one line in client.py. - ty reports call-non-callable / invalid-argument-type on the new class-level-access tests: EndpointMethod.__get__(obj=None) is typed as the descriptor because the overload cannot distinguish the sync vs async owning class, so ty cannot see the callable stub returned at runtime. The three tests deliberately exercise that runtime stub, so they carry targeted ty: ignore suppressions with an explanatory note. Signed-off-by: Max Dubrinsky --- .../src/nemo_platform_plugin/models/client.py | 3 +-- .../tests/client/test_method.py | 20 ++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py index 232ed45576..bf291f0f62 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/models/client.py @@ -94,8 +94,7 @@ def _split_provider_id(deployment: DeploymentLike) -> tuple[str, str]: raise ValueError(f"Deployment '{deployment.name}' has no associated model_provider_id") if "/" not in provider_id: raise ValueError( - f"Deployment '{deployment.name}' has malformed model_provider_id " - f"'{provider_id}'; expected 'workspace/name'" + f"Deployment '{deployment.name}' has malformed model_provider_id '{provider_id}'; expected 'workspace/name'" ) workspace, name = provider_id.split("/", 1) return workspace, name diff --git a/packages/nemo_platform_plugin/tests/client/test_method.py b/packages/nemo_platform_plugin/tests/client/test_method.py index 23c05949db..b3fa724cd9 100644 --- a/packages/nemo_platform_plugin/tests/client/test_method.py +++ b/packages/nemo_platform_plugin/tests/client/test_method.py @@ -48,11 +48,17 @@ def test_class_level_access_returns_a_typed_callable_stub() -> None: def test_class_level_stub_refuses_to_run_unbound() -> None: - """The stub is for introspection only; calling it without an instance errors.""" + """The stub is for introspection only; calling it without an instance errors. + + ty types class-level access as the ``EndpointMethod`` descriptor (the + ``__get__(obj=None)`` overload cannot distinguish the sync vs async owning + class, so it cannot describe the callable stub returned at runtime). These + calls exercise that runtime stub deliberately, hence the suppressions. + """ with pytest.raises(TypeError): - ModelsClient.create_model(workspace="w", body=None) + ModelsClient.create_model(workspace="w", body=None) # ty: ignore[call-non-callable] with pytest.raises(TypeError): - asyncio.run(AsyncModelsClient.create_model(workspace="w", body=None)) + asyncio.run(AsyncModelsClient.create_model(workspace="w", body=None)) # ty: ignore[call-non-callable] def test_mock_spec_against_a_client_class_builds() -> None: @@ -114,8 +120,12 @@ def test_descriptor_carries_endpoint_identity() -> None: def test_signature_is_reachable_at_class_level() -> None: - """The class-level stub is a real function, so ``signature()`` works directly.""" - signature = inspect.signature(AsyncModelsClient.create_model) + """The class-level stub is a real function, so ``signature()`` works directly. + + Typed as the descriptor at class level (see the stub note above), so the + ``signature()`` argument is suppressed; at runtime it is a real function. + """ + signature = inspect.signature(AsyncModelsClient.create_model) # ty: ignore[invalid-argument-type] assert set(signature.parameters) == {"workspace", "body", "exist_ok"} assert all(p.kind is inspect.Parameter.KEYWORD_ONLY for p in signature.parameters.values())