diff --git a/inference_gateway/settings.py b/inference_gateway/settings.py index 9e532bf7..876d49bb 100644 --- a/inference_gateway/settings.py +++ b/inference_gateway/settings.py @@ -127,6 +127,13 @@ # Load maintenance notices to be displayed for individual clusters MAINTENANCE_ERROR_NOTICES = json.loads(os.getenv("MAINTENANCE_ERROR_NOTICES", "{}")) +# Endpoint slug mapping to reroute legacy slugs to their latest value +LEGACY_ENDPOINT_SLUG_MAPPING = json.loads( + os.getenv("LEGACY_ENDPOINT_SLUG_MAPPING", "{}") +) +for key, value in LEGACY_ENDPOINT_SLUG_MAPPING.items(): + LEGACY_ENDPOINT_SLUG_MAPPING[key] = str(value).strip() + # Allowed hosts ALLOWED_HOSTS = textfield_to_strlist(os.getenv("ALLOWED_HOSTS", "*")) diff --git a/resource_server_async/endpoints/endpoint.py b/resource_server_async/endpoints/endpoint.py index fe3c8ea4..22487d70 100644 --- a/resource_server_async/endpoints/endpoint.py +++ b/resource_server_async/endpoints/endpoint.py @@ -7,13 +7,14 @@ from django.forms.models import model_to_dict from django.utils.text import slugify -from inference_gateway.settings import MODEL_DETAILS_KEYS +from inference_gateway.settings import LEGACY_ENDPOINT_SLUG_MAPPING, MODEL_DETAILS_KEYS from resource_server_async.cache import get_redis_client from resource_server_async.rate_limiters import TokenLimiterCheck, TokenRateLimiter from ..auth import check_permission as auth_utils_check_permission from ..errors import ( BatchUnavailable, + EndpointError, EndpointNotFound, Unauthorized, ) @@ -196,6 +197,8 @@ def build_token_limiter( async def load_adapter(cls, cluster: str, framework: str, model: str) -> Self: """Extract the endpoint from the database and return its underlying adapter object.""" endpoint_slug = slugify(f"{cluster} {framework} {model.lower()}") + if endpoint_slug in LEGACY_ENDPOINT_SLUG_MAPPING: + endpoint_slug = LEGACY_ENDPOINT_SLUG_MAPPING[endpoint_slug] if (adapter := _adapter_cache.get(endpoint_slug)) is not None: assert isinstance(adapter, cls) @@ -229,6 +232,10 @@ async def load_adapter(cls, cluster: str, framework: str, model: str) -> Self: raise AssertionError( f"Endpoint adapter {db_endpoint.endpoint_adapter} is not an instance of {cls.__name__}" ) + if endpoint.cluster != cluster or endpoint.framework != framework: + raise EndpointError( + "Endpoint mapping cannot alter 'cluster' or 'framework'." + ) _adapter_cache[endpoint_slug] = endpoint return endpoint diff --git a/resource_server_async/services.py b/resource_server_async/services.py index bad417b5..2d36fb90 100644 --- a/resource_server_async/services.py +++ b/resource_server_async/services.py @@ -288,6 +288,9 @@ async def submit_openai_inference_request( f"endpoint_slug: {endpoint.endpoint_slug} - user: {context.user.username}" ) + # Overwrite model name in case it has been updated via endpoint slug mapping + payload.model = endpoint.model + # Block access if the user is not allowed to use the endpoint endpoint.check_permission(context.user)