⚡️ Speed up function update_response_metadata by 286%

[codeflash-ai]

📄 286% (2.86x) speedup for update_response_metadata in litellm/litellm_core_utils/llm_response_utils/response_metadata.py

⏱️ Runtime : 108 milliseconds → 27.9 milliseconds (best of 148 runs)

📝 Explanation and details

The optimized code achieves a 285% speedup by targeting the most expensive operations identified in the line profiler:

Primary optimization - LRU caching for get_api_base:

• The original code spent 97% of execution time (294ms out of 304ms) calling get_api_base() repeatedly
• Added @lru_cache(maxsize=128) to cache expensive API base lookups using hashable tuples of kwargs
• This dramatically reduces redundant computation when the same model/kwargs combinations are processed

Secondary optimizations:

• Reduced attribute lookups: Extracted logging_obj.model_call_details and logging_obj.caching_details to local variables to avoid repeated dot notation access
• Added local caching: Implemented _hidden_params_cache to avoid repeated lookups of the same hidden parameter keys
• Streamlined conditionals: Simplified the model_info handling to avoid redundant or {} operations

Performance by test case type:

• Basic single calls: 1200-1400% faster (most benefit from API base caching)
• Large scale repeated calls: Up to 3000% faster (cache hits maximize benefit)
• Varied models: Minimal speedup (~0.02%) since each unique model/kwargs combo still requires one expensive call
• Edge cases: 1300-1400% faster due to reduced overhead

The optimization is most effective for workloads with repeated calls using the same model configurations, which is common in production LLM applications where the same models are used across many requests.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 923 Passed
⏪ Replay Tests	✅ 1 Passed
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

import datetime

# imports
import pytest
from litellm.litellm_core_utils.llm_response_utils.response_metadata import \
    update_response_metadata


# Minimal stubs for dependencies (to avoid mocks/stubs, we define real classes)
class HiddenParams(dict):
    # Simulate HiddenParams as a dict for test purposes
    pass

class LiteLLMLoggingObject:
    def __init__(self):
        self.litellm_call_id = "call-id-123"
        self.model_call_details = {}
        self.caching_details = None

    def _response_cost_calculator(self, result, litellm_model_name, router_model_id):
        # Just return a fixed float for testing
        return 42.0
from litellm.litellm_core_utils.llm_response_utils.response_metadata import \
    update_response_metadata


# Helper class for response object
class DummyResponse:
    def __init__(self):
        self._hidden_params = {}

# Helper class for response object with HiddenParams
class DummyResponseHiddenParams:
    def __init__(self):
        self._hidden_params = HiddenParams()

# ------------------- UNIT TESTS -------------------

# 1. Basic Test Cases

def test_basic_metadata_update():
    """Basic: Metadata is set correctly with normal input"""
    result = DummyResponse()
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {"model_info": {"id": "model-abc"}}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)  # 1 second later

    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 182μs -> 13.4μs (1264% faster)

    hidden = result._hidden_params

def test_basic_overhead_time_ms():
    """Basic: Overhead time is calculated if llm_api_duration_ms present"""
    result = DummyResponse()
    logging_obj = LiteLLMLoggingObject()
    logging_obj.model_call_details["llm_api_duration_ms"] = 900
    model = "openai"
    kwargs = {"model_info": {"id": "model-abc"}}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)  # 1 second later

    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 180μs -> 15.2μs (1086% faster)
    hidden = result._hidden_params

def test_basic_cache_overhead():
    """Basic: Overhead time is calculated from cache duration if cache hit"""
    result = DummyResponse()
    logging_obj = LiteLLMLoggingObject()
    logging_obj.caching_details = {"cache_hit": True, "cache_duration_ms": 800}
    model = "openai"
    kwargs = {"model_info": {"id": "model-abc"}}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)  # 1 second later

    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 175μs -> 12.6μs (1288% faster)
    hidden = result._hidden_params

def test_basic_hiddenparams_object():
    """Basic: _hidden_params as HiddenParams object"""
    result = DummyResponseHiddenParams()
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {"model_info": {"id": "model-abc"}}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)

    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 171μs -> 12.3μs (1293% faster)
    hidden = result._hidden_params

# 2. Edge Test Cases

def test_none_result():
    """Edge: If result is None, function returns without error"""
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    # Should not raise
    update_response_metadata(None, logging_obj, model, kwargs, start_time, end_time) # 427ns -> 374ns (14.2% faster)

def test_missing_model_info():
    """Edge: kwargs without model_info should not break"""
    result = DummyResponse()
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {}  # no model_info
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 185μs -> 204μs (9.34% slower)
    hidden = result._hidden_params

def test_missing_llm_api_duration_ms():
    """Edge: No llm_api_duration_ms, overhead time should not be set"""
    result = DummyResponse()
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 173μs -> 12.2μs (1319% faster)
    hidden = result._hidden_params

def test_cache_hit_false():
    """Edge: cache_hit is False, should not set overhead time from cache"""
    result = DummyResponse()
    logging_obj = LiteLLMLoggingObject()
    logging_obj.caching_details = {"cache_hit": False, "cache_duration_ms": 800}
    model = "openai"
    kwargs = {}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 176μs -> 11.6μs (1422% faster)
    hidden = result._hidden_params

def test_no_hidden_params_attribute():
    """Edge: result without _hidden_params attribute does not raise"""
    class NoHiddenParamsObj:
        pass
    result = NoHiddenParamsObj()
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    # Should not raise
    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 181μs -> 12.2μs (1385% faster)
    # No assertion needed, just check no error

def test_negative_time_interval():
    """Edge: Negative time interval should produce negative response_ms"""
    result = DummyResponse()
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 173μs -> 12.2μs (1329% faster)
    hidden = result._hidden_params

def test_empty_kwargs():
    """Edge: Empty kwargs should not break"""
    result = DummyResponse()
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 172μs -> 11.3μs (1418% faster)
    hidden = result._hidden_params

def test_additional_headers_merging():
    """Edge: _hidden_params with additional_headers should merge correctly"""
    result = DummyResponse()
    result._hidden_params = {"additional_headers": {"x-foo": "bar"}}
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 170μs -> 13.0μs (1218% faster)
    hidden = result._hidden_params

# 3. Large Scale Test Cases

def test_large_scale_many_metadata_updates():
    """Large Scale: Update many response objects in a loop"""
    responses = [DummyResponse() for _ in range(500)]
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {"model_info": {"id": "model-abc"}}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    for result in responses:
        update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 56.7ms -> 1.78ms (3095% faster)
    for result in responses:
        hidden = result._hidden_params

def test_large_scale_varied_models():
    """Large Scale: Update with varied model names and model_info"""
    responses = [DummyResponse() for _ in range(100)]
    logging_obj = LiteLLMLoggingObject()
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    for i, result in enumerate(responses):
        model = f"model{i}"
        kwargs = {"model_info": {"id": f"id{i}"}}
        update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 11.8ms -> 11.8ms (0.018% faster)
    for i, result in enumerate(responses):
        hidden = result._hidden_params

def test_large_scale_with_hiddenparams_object():
    """Large Scale: Many updates with HiddenParams object"""
    responses = [DummyResponseHiddenParams() for _ in range(100)]
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {"model_info": {"id": "model-abc"}}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 1)
    for result in responses:
        update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 11.5ms -> 373μs (2979% faster)
    for result in responses:
        hidden = result._hidden_params

def test_large_scale_cache_and_overhead():
    """Large Scale: Many responses with cache hit and overhead calculation"""
    responses = [DummyResponse() for _ in range(100)]
    logging_obj = LiteLLMLoggingObject()
    model = "openai"
    kwargs = {"model_info": {"id": "model-abc"}}
    start_time = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end_time = datetime.datetime(2024, 1, 1, 0, 0, 2)  # 2 seconds
    for result in responses:
        # Each response gets a unique cache duration
        logging_obj.caching_details = {"cache_hit": True, "cache_duration_ms": 1000}
        update_response_metadata(result, logging_obj, model, kwargs, start_time, end_time) # 11.5ms -> 397μs (2805% faster)
    for result in responses:
        hidden = result._hidden_params
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.
#------------------------------------------------
import datetime

# imports
import pytest
from litellm.litellm_core_utils.llm_response_utils.response_metadata import \
    update_response_metadata

# --- Minimal stubs/mocks for dependencies ---

class HiddenParams(dict):
    """A minimal stub for HiddenParams, acting like a dict but with attribute access."""
    def __getattr__(self, item):
        try:
            return self[item]
        except KeyError:
            return None
    def __setattr__(self, key, value):
        self[key] = value

class LiteLLMLoggingObject:
    """A minimal stub for LiteLLMLoggingObject."""
    def __init__(
        self,
        litellm_call_id="callid123",
        model_call_details=None,
        caching_details=None,
        response_cost=42.5,
    ):
        self.litellm_call_id = litellm_call_id
        self.model_call_details = model_call_details or {}
        self.caching_details = caching_details
        self._response_cost = response_cost
    def _response_cost_calculator(self, result, litellm_model_name, router_model_id):
        return self._response_cost
from litellm.litellm_core_utils.llm_response_utils.response_metadata import \
    update_response_metadata

# --- Test helpers ---

class DummyResult:
    """A minimal result object with _hidden_params and optional _response_ms."""
    def __init__(self, hidden_params=None):
        self._hidden_params = hidden_params if hidden_params is not None else {}

# --- Unit tests ---

# 1. Basic Test Cases

def test_basic_hidden_params_update():
    """Test that update_response_metadata sets basic hidden params correctly."""
    result = DummyResult()
    logging_obj = LiteLLMLoggingObject()
    model = "test-model"
    kwargs = {"model_info": {"id": "model-123"}}
    start = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start, end) # 221μs -> 222μs (0.409% slower)
    # Check hidden params keys
    hp = result._hidden_params

def test_basic_timing_and_overhead():
    """Test that response time and overhead are set when llm_api_duration_ms is present."""
    result = DummyResult()
    logging_obj = LiteLLMLoggingObject(model_call_details={"llm_api_duration_ms": 800})
    model = "test-model"
    kwargs = {}
    start = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start, end) # 192μs -> 191μs (0.455% faster)
    hp = result._hidden_params


def test_result_is_none():
    """Test that update_response_metadata does nothing if result is None."""
    # Should not raise
    update_response_metadata(None, LiteLLMLoggingObject(), "m", {}, datetime.datetime.now(), datetime.datetime.now()) # 513ns -> 500ns (2.60% faster)


def test_missing_model_info_and_headers():
    """Test that missing model_info and additional_headers are handled gracefully."""
    result = DummyResult()
    logging_obj = LiteLLMLoggingObject()
    model = "test-model"
    kwargs = {}  # no model_info
    start = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start, end) # 188μs -> 12.5μs (1401% faster)
    hp = result._hidden_params

def test_caching_details_cache_hit():
    """Test that caching_details with cache_hit=True sets overhead correctly."""
    result = DummyResult()
    caching_details = {"cache_hit": True, "cache_duration_ms": 900}
    logging_obj = LiteLLMLoggingObject(caching_details=caching_details)
    model = "test-model"
    kwargs = {}
    start = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start, end) # 182μs -> 12.2μs (1397% faster)
    hp = result._hidden_params

def test_no_hidden_params_attribute():
    """Test that update_response_metadata works if result initially lacks _hidden_params."""
    class NoHiddenParams:
        pass
    result = NoHiddenParams()
    setattr(result, "_hidden_params", {})
    logging_obj = LiteLLMLoggingObject()
    model = "test-model"
    kwargs = {}
    start = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start, end) # 182μs -> 11.8μs (1454% faster)
    hp = getattr(result, "_hidden_params")

def test_negative_duration():
    """Test that negative durations are handled (should still compute, but negative ms)."""
    result = DummyResult()
    logging_obj = LiteLLMLoggingObject()
    model = "test-model"
    kwargs = {}
    start = datetime.datetime(2024, 1, 1, 0, 0, 1)
    end = datetime.datetime(2024, 1, 1, 0, 0, 0)
    update_response_metadata(result, logging_obj, model, kwargs, start, end) # 180μs -> 11.7μs (1440% faster)
    hp = result._hidden_params

def test_large_cost_value():
    """Test that a very large response_cost is handled."""
    result = DummyResult()
    logging_obj = LiteLLMLoggingObject(response_cost=10**9)
    model = "big-model"
    kwargs = {}
    start = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start, end) # 181μs -> 212μs (14.6% slower)
    hp = result._hidden_params

def test_hiddenparams_attribute_access():
    """Test that HiddenParams supports attribute and item access."""
    hp = HiddenParams()
    hp.foo = "bar"
    hp["baz"] = "qux"

# 3. Large Scale Test Cases

def test_many_hidden_params_keys():
    """Test updating with a large number of keys in hidden params."""
    initial = {f"key{i}": i for i in range(500)}
    result = DummyResult(hidden_params=initial.copy())
    logging_obj = LiteLLMLoggingObject()
    model = "large-model"
    kwargs = {"model_info": {"id": "large-001"}}
    start = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end = datetime.datetime(2024, 1, 1, 0, 0, 2)
    update_response_metadata(result, logging_obj, model, kwargs, start, end) # 198μs -> 197μs (0.314% faster)
    hp = result._hidden_params
    # All original keys remain, and new ones are added/updated
    for i in range(500):
        pass

def test_large_cache_duration():
    """Test with a large cache_duration_ms value."""
    result = DummyResult()
    caching_details = {"cache_hit": True, "cache_duration_ms": 999}
    logging_obj = LiteLLMLoggingObject(caching_details=caching_details)
    model = "test-model"
    kwargs = {}
    start = datetime.datetime(2024, 1, 1, 0, 0, 0)
    end = datetime.datetime(2024, 1, 1, 0, 0, 1)
    update_response_metadata(result, logging_obj, model, kwargs, start, end) # 185μs -> 13.2μs (1303% faster)
    hp = result._hidden_params

def test_massive_number_of_calls():
    """Test calling update_response_metadata many times to check for memory leaks or state issues."""
    for i in range(100):  # Keep under 1000 for performance
        result = DummyResult()
        logging_obj = LiteLLMLoggingObject(litellm_call_id=f"id{i}")
        model = f"model{i}"
        kwargs = {"model_info": {"id": f"mid{i}"}}
        start = datetime.datetime(2024, 1, 1, 0, 0, 0)
        end = datetime.datetime(2024, 1, 1, 0, 0, 1)
        update_response_metadata(result, logging_obj, model, kwargs, start, end) # 11.8ms -> 11.9ms (0.232% slower)
        hp = result._hidden_params

⏪ Replay Tests and Runtime

Test File::Test Function	Original ⏱️	Optimized ⏱️	Speedup
test_pytest_teststest_litellmresponseslitellm_completion_transformationtest_litellm_completion_responses___replay_test_0.py::test_litellm_litellm_core_utils_llm_response_utils_response_metadata_update_response_metadata	1.50μs	1.49μs	0.670%✅

To edit these changes git checkout codeflash/optimize-update_response_metadata-mhl4ji2j and push.