Fixes from reviews

Author: markurtz

src/guidellm/benchmark/benchmarker.py

@@ -13,7 +13,7 @@
 import uuid
 from abc import ABC
 from collections.abc import AsyncIterator, Iterable
-from typing import Any, Generic
+from typing import Generic
 
 from guidellm.benchmark.profile import Profile
 from guidellm.benchmark.progress import BenchmarkerProgress

src/guidellm/benchmark/entrypoints.py

@@ -309,7 +309,7 @@ async def resolve_request_loader(
 
 async def resolve_profile(
     profile: StrategyType | ProfileType | Profile,
-    rate: float | list[float] | None,
+    rate: list[float] | None,
     random_seed: int,
     constraints: MutableMapping[str, ConstraintInitializer | Any],
     max_seconds: int | float | None,
@@ -355,10 +355,9 @@ async def resolve_profile(
         if val is not None:
             constraints[key] = val
     if not isinstance(profile, Profile):
-        rate_list: list[float] | None = [rate] if isinstance(rate, float) else rate
         profile = Profile.create(
             rate_type=profile,
-            rate=rate_list,
+            rate=rate,
             random_seed=random_seed,
             constraints={**constraints},
         )

src/guidellm/benchmark/outputs/console.py

@@ -94,9 +94,9 @@ def add_stats(
         precision: int = 1,
     ):
         """
-        Add statistical summary columns (mean and standard deviation) for a metric.
+        Add statistical summary columns (mean and p95) for a metric.
 
-        Creates paired mean/stddev columns automatically and appends values from the
+        Creates paired mean/p95 columns automatically and appends values from the
         specified status category of the distribution summary.
 
         :param stats: Distribution summary containing status-specific statistics
@@ -111,16 +111,16 @@ def add_stats(
             self[f"{key}_mean"] = ConsoleTableColumn(
                 group=group, name=name, units="Mean", precision=precision
             )
-            self[f"{key}_stddev"] = ConsoleTableColumn(
-                group=group, name=name, units="Std", precision=precision
+            self[f"{key}_p95"] = ConsoleTableColumn(
+                group=group, name=name, units="p95", precision=precision
             )
 
         status_stats: DistributionSummary | None = (
             getattr(stats, status) if stats else None
         )
         self[f"{key}_mean"].values.append(status_stats.mean if status_stats else None)
-        self[f"{key}_stddev"].values.append(
-            status_stats.std_dev if status_stats else None
+        self[f"{key}_p95"].values.append(
+            status_stats.percentiles.p95 if status_stats else None
         )
 
     def get_table_data(self) -> tuple[list[list[str]], list[list[str]]]:

src/guidellm/benchmark/progress.py

@@ -348,9 +348,9 @@ class _GenerativeProgressTaskState:
     request_concurrency: float = 0.0
     requests_per_second: float = 0.0
     request_latency: float = 0.0
-    output_tokens: int = 0
+    output_tokens: float = 0
     output_tokens_rate: float = 0.0
-    prompt_tokens: int = 0
+    prompt_tokens: float = 0
     total_tokens_rate: float = 0.0
     time_to_first_token: float = 0.0
     inter_token_latency: float = 0.0
@@ -588,13 +588,9 @@ def update(
             request_latency=accumulator.completed_metrics.request_latency.mean,
         )
         self._update_token_stats(
-            output_tokens=int(
-                accumulator.completed_metrics.total_tokens.value_sum or 0
-            ),
+            output_tokens=accumulator.completed_metrics.total_tokens.mean,
             output_tokens_rate=accumulator.completed_metrics.output_tokens.rate_per_second,
-            prompt_tokens=int(
-                accumulator.completed_metrics.input_tokens.value_sum or 0
-            ),
+            prompt_tokens=accumulator.completed_metrics.input_tokens.mean,
             total_tokens_rate=accumulator.completed_metrics.total_tokens.rate_per_second,
             time_to_first_token=accumulator.completed_metrics.time_to_first_token_ms.mean,
             inter_token_latency=accumulator.completed_metrics.inter_token_latency_ms.mean,
@@ -621,13 +617,9 @@ def complete(self, benchmark: GenerativeBenchmark):
             request_latency=benchmark.metrics.request_latency.successful.mean,
         )
         self._update_token_stats(
-            output_tokens=int(
-                benchmark.metrics.output_token_count.successful.mean or 0
-            ),
+            output_tokens=benchmark.metrics.output_token_count.successful.mean,
             output_tokens_rate=benchmark.metrics.output_tokens_per_second.successful.mean,
-            prompt_tokens=int(
-                benchmark.metrics.prompt_token_count.successful.mean or 0
-            ),
+            prompt_tokens=benchmark.metrics.prompt_token_count.successful.mean,
             total_tokens_rate=benchmark.metrics.tokens_per_second.successful.mean,
             time_to_first_token=(
                 benchmark.metrics.time_to_first_token_ms.successful.mean
@@ -682,9 +674,9 @@ def _update_request_stats(
 
     def _update_token_stats(
         self,
-        output_tokens: int | None = None,
+        output_tokens: float | None = None,
         output_tokens_rate: float | None = None,
-        prompt_tokens: int | None = None,
+        prompt_tokens: float | None = None,
         total_tokens_rate: float | None = None,
         time_to_first_token: float | None = None,
         inter_token_latency: float | None = None,

src/guidellm/benchmark/schemas/generative/entrypoints.py

@@ -17,7 +17,16 @@
 from typing import Any, Literal
 
 import yaml
-from pydantic import ConfigDict, Field, model_serializer
+from pydantic import (
+    AliasChoices,
+    AliasGenerator,
+    ConfigDict,
+    Field,
+    ValidationError,
+    ValidatorFunctionWrapHandler,
+    field_validator,
+    model_serializer,
+)
 from torch.utils.data import Sampler
 from transformers import PreTrainedTokenizerBase
 
@@ -101,9 +110,8 @@ def create(
                 scenario_data = scenario_data["args"]
             constructor_kwargs.update(scenario_data)
 
-        for key, value in kwargs.items():
-            if value != cls.get_default(key):
-                constructor_kwargs[key] = value
+        # Apply overrides from kwargs
+        constructor_kwargs.update(kwargs)
 
         return cls.model_validate(constructor_kwargs)
 
@@ -138,6 +146,14 @@ def get_default(cls: type[BenchmarkGenerativeTextArgs], field: str) -> Any:
         use_enum_values=True,
         from_attributes=True,
         arbitrary_types_allowed=True,
+        validate_by_alias=True,
+        validate_by_name=True,
+        alias_generator=AliasGenerator(
+            # Support field names with hyphens
+            validation_alias=lambda field_name: AliasChoices(
+                field_name, field_name.replace("_", "-")
+            ),
+        ),
     )
 
     # Required
@@ -151,7 +167,7 @@ def get_default(cls: type[BenchmarkGenerativeTextArgs], field: str) -> Any:
     profile: StrategyType | ProfileType | Profile = Field(
         default="sweep", description="Benchmark profile or scheduling strategy type"
     )
-    rate: float | list[float] | None = Field(
+    rate: list[float] | None = Field(
         default=None, description="Request rate(s) for rate-based scheduling"
     )
     # Backend configuration
@@ -187,6 +203,12 @@ def get_default(cls: type[BenchmarkGenerativeTextArgs], field: str) -> Any:
     data_request_formatter: RequestFormatter | dict[str, str] | str = Field(
         default="chat_completions",
         description="Request formatting preprocessor or template name",
+        validation_alias=AliasChoices(
+            "data_request_formatter",
+            "data-request-formatter",
+            "request_type",
+            "request-type",
+        ),
     )
     data_collator: Callable | Literal["generative"] | None = Field(
         default="generative", description="Data collator for batch processing"
@@ -243,6 +265,26 @@ def get_default(cls: type[BenchmarkGenerativeTextArgs], field: str) -> Any:
         default=None, description="Maximum global error rate (0-1) before stopping"
     )
 
+    @field_validator("data", "data_args", "rate", mode="wrap")
+    @classmethod
+    def single_to_list(
+        cls, value: Any, handler: ValidatorFunctionWrapHandler
+    ) -> list[Any]:
+        """
+        Ensures field is always a list.
+
+        :param value: Input value for the 'data' field
+        :return: List of data sources
+        """
+        try:
+            return handler(value)
+        except ValidationError as err:
+            # If validation fails, try wrapping the value in a list
+            if err.errors()[0]["type"] == "list_type":
+                return handler([value])
+            else:
+                raise
+
     @model_serializer
     def serialize_model(self) -> dict[str, Any]:
         """

src/guidellm/scheduler/worker_group.py

@@ -228,11 +228,11 @@ async def create_processes(self):
 
             worker = WorkerProcess[RequestT, ResponseT](
                 worker_index=rank,
-                messaging=self.messaging.create_worker_copy(
+                messaging=self.messaging.create_worker_copy(  # type: ignore[arg-type]
                     worker_index=rank,
                     max_buffer_send_size=None,
                     max_buffer_receive_size=per_proc_max_buffer_size,
-                ),  # The non-group worker lacks the SchedulerState type. Type err.
+                ),
                 backend=self.backend,
                 strategy=self.strategy,
                 async_limit=async_limit,

src/guidellm/schemas/request_stats.py

@@ -266,12 +266,10 @@ def prompt_tokens_timing(self) -> tuple[float, float] | None:
             # no end time, can't compute
             return None
 
-        return [
-            (
-                self.first_token_iteration or self.request_end_time,
-                self.prompt_tokens or 0.0,
-            )
-        ]
+        return (
+            self.first_token_iteration or self.request_end_time,
+            self.prompt_tokens or 0.0,
+        )
 
     @property
     def output_tokens_timings(self) -> list[tuple[float, float]]:
@@ -332,4 +330,4 @@ def total_tokens_timings(self) -> list[tuple[float, float]]:
         prompt_timings = self.prompt_tokens_timing
         output_timings = self.output_tokens_timings
 
-        return (prompt_timings or []) + output_timings
+        return ([prompt_timings] if prompt_timings else []) + output_timings

src/guidellm/schemas/statistics.py

@@ -714,7 +714,7 @@ def from_values_function(
         def _extract_values(
             _objs: Sequence[FunctionObjT],
         ) -> Sequence[float | tuple[float, float]]:
-            _outputs = []
+            _outputs: list[float | tuple[float, float]] = []
             for _obj in _objs:
                 if (_result := function(_obj)) is None:
                     continue
@@ -830,7 +830,7 @@ def rate_distribution_from_timings_function(
         def _extract_values(
             _objs: Sequence[FunctionObjT],
         ) -> Sequence[float | tuple[float, float]]:
-            _outputs = []
+            _outputs: list[float | tuple[float, float]] = []
             for _obj in _objs:
                 if (_result := function(_obj)) is None:
                     continue
@@ -955,7 +955,7 @@ def concurrency_distribution_from_timings_function(
         def _extract_values(
             _objs: Sequence[FunctionObjT],
         ) -> Sequence[tuple[float, float] | tuple[float, float, float]]:
-            _outputs = []
+            _outputs: list[tuple[float, float] | tuple[float, float, float]] = []
             for _obj in _objs:
                 if (_result := function(_obj)) is None:
                     continue
@@ -979,10 +979,13 @@ def _extract_values(
     @classmethod
     def _combine_status_arrays(
         cls,
-        successful: Sequence[float] | np.ndarray,
-        incomplete: Sequence[float] | np.ndarray,
-        errored: Sequence[float] | np.ndarray,
-        num_values_per_item: int,
+        successful: Sequence[float | tuple[float, float] | tuple[float, float, float]]
+        | np.ndarray,
+        incomplete: Sequence[float | tuple[float, float] | tuple[float, float, float]]
+        | np.ndarray,
+        errored: Sequence[float | tuple[float, float] | tuple[float, float, float]]
+        | np.ndarray,
+        num_values_per_item: Literal[2, 3],
     ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
         successful_array = DistributionSummary._to_weighted_ndarray(  # noqa: SLF001
             successful, num_values_per_item=num_values_per_item

src/guidellm/settings.py

@@ -162,7 +162,7 @@ class Settings(BaseSettings):
     preferred_output_tokens_source: Literal["request", "response"] = "response"
     preferred_backend: Literal["openai"] = "openai"
     preferred_route: Literal["text_completions", "chat_completions"] = (
-        "text_completions"
+        "chat_completions"
     )
     openai: OpenAISettings = OpenAISettings()