diff --git a/.gitignore b/.gitignore index 0a87280b..408edca1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ target/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # Added by cargo @@ -27,4 +27,4 @@ target/ .env /scripts .vscode -.cargo \ No newline at end of file +.cargo diff --git a/crates/dspy-rs/src/core/lm/config.rs b/crates/dspy-rs/src/core/lm/config.rs index 8e23f442..102be374 100644 --- a/crates/dspy-rs/src/core/lm/config.rs +++ b/crates/dspy-rs/src/core/lm/config.rs @@ -2,27 +2,39 @@ use bon::Builder; use serde_json::Value; use std::collections::HashMap; +/// Tunable inference parameters applied to each [`LM::call`]. #[derive(Clone, Debug, Builder)] pub struct LMConfig { + /// Default model identifier. Accepts `provider/model` to infer base URL. #[builder(default = "gpt-4o-mini".to_string())] pub model: String, + /// Sampling temperature. Higher values increase randomness. #[builder(default = 0.7)] pub temperature: f32, + /// Nucleus sampling parameter (`top_p`). Set either temperature or `top_p`. #[builder(default = 0.0)] pub top_p: f32, + /// Maximum tokens requested for the completion. #[builder(default = 512)] pub max_tokens: u32, + /// Reserved for providers that differentiate prompt vs. completion limits. #[builder(default = 512)] pub max_completion_tokens: u32, + /// Number of completions to request per call. #[builder(default = 1)] pub n: u8, + /// Presence penalty forwarded to compatible providers. #[builder(default = 0.0)] pub presence_penalty: f32, + /// Frequency penalty forwarded to compatible providers. #[builder(default = 0.0)] pub frequency_penalty: f32, + /// Optional deterministic seed when the provider supports it. #[builder(default = 42)] pub seed: i64, + /// Token-level logit adjustments keyed by provider-specific IDs. pub logit_bias: Option>, + /// Enables the shared response cache and history surface. #[builder(default = true)] pub cache: bool, } diff --git a/crates/dspy-rs/src/core/lm/mod.rs b/crates/dspy-rs/src/core/lm/mod.rs index 92299869..1c9f5113 100644 --- a/crates/dspy-rs/src/core/lm/mod.rs +++ b/crates/dspy-rs/src/core/lm/mod.rs @@ -16,10 +16,18 @@ use secrecy::{ExposeSecret, SecretString}; use std::{collections::HashMap, sync::Arc}; use tokio::sync::Mutex; +/// A single completion returned by [`LM::call`]. +/// +/// Captures the assistant reply (`output`), the provider token accounting +/// (`usage`), and the final chat transcript (`chat`) so higher-level modules +/// can inspect the full exchange. #[derive(Clone, Debug)] pub struct LMResponse { + /// Assistant message chosen by the provider. pub output: Message, + /// Token usage reported by the provider for this call. pub usage: LmUsage, + /// Chat history including the freshly appended assistant response. pub chat: Chat, } @@ -38,16 +46,25 @@ fn get_base_url_by_provider(provider: &str) -> &str { } } +/// OpenAI-compatible language model client used throughout DSRs. +/// +/// `LM` owns provider credentials, request configuration, and optional +/// response caching. Builders are cheap to clone; clones share the same HTTP +/// client and cache via `Arc` so they remain lightweight for concurrent use. #[derive(Builder)] #[builder(finish_fn(vis = "", name = build_internal))] pub struct LM { + /// Provider API credential stored as a [`SecretString`]. #[builder(getter)] pub api_key: SecretString, + /// Base URL for the OpenAI-compatible endpoint. #[builder(default = "https://api.openai.com/v1".to_string(), getter)] pub base_url: String, + /// Model inference settings applied to each call. #[builder(default = LMConfig::default(), getter)] pub config: LMConfig, client: Option>, + /// Optional shared cache used to deduplicate identical requests. pub cache_handler: Option>>, } @@ -66,6 +83,8 @@ impl Clone for LM { use l_m_builder::{IsSet, IsUnset, State}; impl LMBuilder { + /// Finalizes construction of an [`LM`], initializing the HTTP client and + /// optional response cache. pub async fn build(self) -> LM where S::ApiKey: IsSet, @@ -96,6 +115,11 @@ impl LMBuilder { } impl LM { + /// Executes a chat completion against the configured provider. + /// + /// `messages` must already be formatted as OpenAI-compatible chat turns. + /// The call returns an [`LMResponse`] containing the assistant output, + /// token usage, and chat history including the new response. pub async fn call(&self, messages: Chat) -> Result { let request_messages = messages.get_async_openai_messages(); @@ -138,6 +162,9 @@ impl LM { }) } + /// Returns the `n` most recent cached calls. + /// + /// Panics if caching is disabled for this `LM`. pub async fn inspect_history(&self, n: usize) -> Vec { self.cache_handler .as_ref() @@ -150,17 +177,23 @@ impl LM { } } +/// In-memory LM used for deterministic tests and examples. #[derive(Clone, Builder, Default)] pub struct DummyLM { + /// Synthetic API key; unused but mirrors [`LM`]. pub api_key: SecretString, + /// Base URL retained for parity with [`LM`]. #[builder(default = "https://api.openai.com/v1".to_string())] pub base_url: String, + /// Static configuration applied to stubbed responses. #[builder(default = LMConfig::default())] pub config: LMConfig, + /// Cache backing storage shared with the real implementation. pub cache_handler: Option>>, } impl DummyLM { + /// Creates a new [`DummyLM`] with an enabled in-memory cache. pub async fn new() -> Self { let cache_handler = Arc::new(Mutex::new(ResponseCache::new().await)); Self { @@ -171,6 +204,10 @@ impl DummyLM { } } + /// Mimics [`LM::call`] without hitting a remote provider. + /// + /// The provided `prediction` becomes the assistant output and is inserted + /// into the shared cache when caching is enabled. pub async fn call( &self, example: Example, @@ -215,6 +252,7 @@ impl DummyLM { }) } + /// Returns cached entries just like [`LM::inspect_history`]. pub async fn inspect_history(&self, n: usize) -> Vec { self.cache_handler .as_ref() diff --git a/docs/docs/building-blocks/lm.mdx b/docs/docs/building-blocks/lm.mdx index 547f027e..45defd2f 100644 --- a/docs/docs/building-blocks/lm.mdx +++ b/docs/docs/building-blocks/lm.mdx @@ -4,46 +4,74 @@ description: 'Configure and integrate language models in your pipelines' icon: 'arrow-down-a-z' --- -DSRs treats the Language Model (`LM`) as a first–class, configurable client for chat-style inference. This page explains what an LM is in DSRs, how it’s structured in Rust terms, and how it cooperates with other building blocks. +The Language Model (`LM`) struct is a configurable client for calling LLM providers, with built-in caching and history tracking. + +This page explains what an LM is in DSRs, how it’s structured in Rust terms, and how it cooperates with other building blocks. ## What is an LM? -- **Purpose:** Encapsulates a provider client (e.g., OpenAI) and model-level settings, and executes chat completions. -- **Rust shape:** `LM` is a clonable struct built via a builder (`LM::builder()`), holding `LMConfig` and an internal client. -- **Async-first:** `LM::call(...)` is `async` and returns a `(Message, LmUsage)` pair. -- **History:** Each call is recorded; inspect recent calls via `lm.inspect_history(n)`. +The `LM` struct is a thin wrapper around OpenAI-compatible API clients, with built-in support for multiple providers. + +It handles three core responsibilities: + +1. **Configuration** - Stores provider credentials, model selection, and inference parameters (eg: temperature) + +2. **API Execution** - Takes pre-formatted `Chat` messages and executes HTTP calls to the LLM provider + +3. **Response Caching** - Optionally stores input/output pairs to avoid duplicate API calls + + +### Structure + +`LM` is built using the builder pattern and holds: + - `api_key` - Provider API credentials (stored as `SecretString`) + - `base_url` - API endpoint URL + - `config` - An `LMConfig` with model settings + - `client` - Internal HTTP client (OpenAI-compatible) + - `cache_handler` - Optional response cache (enabled by default) + +Cloning an `LM` is cheap - clones share the same HTTP client and cache via `Arc`, making them ideal for concurrent use. + + +## Where it fits -## Key types +You rarely call `LM` directly—It's the lowest-level DSRs primitive. Instead, a `Predictor` uses an `Adapter` to format a `Signature` and call the LM. This keeps business logic (your task) separate from transport (the model client). -- **`LM` (struct):** Holds `api_key`, `base_url`, `config`, and a `history` of calls. Implements `Clone`. -- **`LMConfig` (struct):** Builder-driven config (e.g., `model`, `temperature`, `max_tokens`). -- **`Chat`, `Message` (structs):** Internal chat abstraction used by adapters to format/parse messages. -- **`Adapter` (trait) and `ChatAdapter` (impl):** Orchestrate how a `Signature` + `Example` become a `Chat`, and parse the model’s response back to a `Prediction`. ## Construction and configuration ```rust use dsrs::{LM, LMConfig}; -let lm = LM::builder() - .api_key(std::env::var("OPENAI_API_KEY")?.into()) - .config( - LMConfig::builder() - .model("gpt-4o-mini".to_string()) - .temperature(0.7) - .max_tokens(512) - .build() - ) - .build(); +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let lm = LM::builder() + .api_key(std::env::var("OPENAI_API_KEY")?.into()) + .config( + LMConfig::builder() + .model("gpt-4o-mini".to_string()) + .temperature(0.7) + .max_tokens(512) + .build() + ) + .build() + .await; + + // use lm here + Ok(()) +} ``` -- **Builder pattern:** Idiomatic in Rust; keeps construction explicit and typed. -- **Clone semantics:** `LM` implements `Clone`—cloning copies config and history for that instance. +- **Clone semantics:** `LM` implements `Clone`; clones share the underlying client and cache via `Arc`, so they see the same history while carrying their own config copy. + +## API Reference + +You can browse the full `LM` module reference on [docs.rs](https://docs.rs/dspy-rs/latest/dspy_rs/core/lm/index.html). ## Global vs explicit usage - **Global:** `configure(lm.clone(), ChatAdapter::default())` sets the process-wide default used by predictors. -- **Explicit:** Some APIs accept a mutable `&mut LM` if you prefer local control: `predictor.forward_with_config(inputs, &mut lm).await`. +- **Explicit:** Wrap the model in an `Arc` when you want to override the global instance: `let shared = Arc::new(lm); predictor.forward_with_config(inputs, Arc::clone(&shared)).await`. ## Async execution and sync entry @@ -80,13 +108,82 @@ fn main() -> anyhow::Result<()> { ## Inspecting history ```rust -let history = lm.inspect_history(3); +let history = lm.inspect_history(3).await; for entry in history { - println!("Model: {} | Output: {}", entry.config.model, entry.output.content()); + println!("Prompt: {}", entry.prompt); + println!("Prediction: {:?}", entry.prediction.data); } ``` -## Where it fits +> `inspect_history` requires caching to be enabled on the LM; otherwise no history is recorded. + +## LMConfig reference + +`LMConfig` centralizes inference settings. All fields have sensible defaults, so you can override only what you need. The builder exposes setters for each field. + +| Field | Type | Default | Notes | +|-------------------------|----------------------------------|-----------------|---------------------------------------------------------------------------------| +| `model` | `String` | `"gpt-4o-mini"` | Supports bare IDs or `provider/model` prefixed IDs; affects base URL inference. | +| `temperature` | `f32` | `0.7` | Higher values increase randomness. | +| `top_p` | `f32` | `0.0` | Nucleus sampling; set one of `temperature` or `top_p`. | +| `max_tokens` | `u32` | `512` | Upper bound on completion tokens. | +| `max_completion_tokens` | `u32` | `512` | Not currently sent to providers; Reserved for future use. | +| `presence_penalty` | `f32` | `0.0` | Penalizes repeated tokens globally. | +| `frequency_penalty` | `f32` | `0.0` | Penalizes repeated tokens proportionally; ignored for Gemini models. | +| `seed` | `i64` | `42` | Optional seed for deterministic sampling; ignored for Gemini models. | +| `logit_bias` | `Option>` | `None` | Token-level logit adjustments; ignored for Gemini models. | +| `cache` | `bool` | `true` | Enables the shared `ResponseCache` and `inspect_history` support. | + + + +```rust +let config = LMConfig::builder() + .model("anthropic/claude-4.5-sonnet".into()) + .temperature(0.3) + .max_tokens(1_024) + .cache(true) + .build(); +``` + +> Tip: stick to either `temperature` or `top_p`; providers often ignore one when both are set. + +### Provider Support & Base URLs + +DSRs supports multiple LLM providers through automatic endpoint detection. When you pass a `provider/model` string, the builder splits it, rewrites `config.model` to just the model id, and swaps `base_url` to the matching OpenAI-compatible endpoint. Bare model names keep the default OpenAI base URL, and any unknown prefix falls back to OpenRouter. + +| Prefix | Base URL | +|--------------|-----------------------------------------------------------| +| `openai` | `https://api.openai.com/v1` | +| `anthropic` | `https://api.anthropic.com/v1` | +| `google` | `https://generativelanguage.googleapis.com/v1beta/openai` | +| `cohere` | `https://api.cohere.ai/compatibility/v1` | +| `groq` | `https://api.groq.com/openai/v1` | +| `openrouter` | `https://openrouter.ai/api/v1` | +| `qwen` | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | +| `together` | `https://api.together.xyz/v1` | +| `xai` | `https://api.x.ai/v1` | + +You can still override the base URL manually via the builder if you need a self-hosted proxy. + +Use the `provider/model` format to target specific hosts: + +```rust +// Anthropic +.model("anthropic/claude-4-5-sonnet".to_string()) + +// Google +.model("google/gemini-2.0-flash".to_string()) + +// Groq +.model("groq/mixtral-8x7b-32768".to_string()) + +// OpenAI (or just use model name directly) +.model("openai/gpt-4.1".to_string()) +.model("gpt-4.1-mini".to_string()) // defaults to OpenAI +``` + +Supported prefixes: `openai`, `anthropic`, `google`, `cohere`, `groq`, `openrouter`, `qwen`, `together`, `xai`. Unrecognised prefixes default to OpenRouter. + +### Gemini compatibility -- You rarely call `LM` directly; instead, a `Predictor` uses an `Adapter` to format a `Signature` and call the LM. -- This keeps business logic (your task) separate from transport (the model client), matching Rust’s trait-driven composition style. +Gemini models (names starting with `gemini-`) reject several OpenAI parameters. DSRs automatically omits `frequency_penalty`, `seed`, and `logit_bias` when the configured model begins with `gemini-`, so you can keep the same `LMConfig` across providers without triggering errors. Other settings such as `temperature`, `top_p`, `max_tokens`, and `presence_penalty` are still forwarded. diff --git a/docs/docs/getting-started/quickstart.mdx b/docs/docs/getting-started/quickstart.mdx index a1852f5c..aa97e958 100644 --- a/docs/docs/getting-started/quickstart.mdx +++ b/docs/docs/getting-started/quickstart.mdx @@ -55,16 +55,18 @@ into a prompt that the LM can follow to complete the task. use dspy_rs::{configure, ChatAdapter, LM, LMConfig}; use std::env; -fn main() -> Result<(), anyhow::Error> { - //Define a config for the LM +#[tokio::main] +async fn main() -> Result<(), anyhow::Error> { + // Define a config for the LM let config = LMConfig::builder() .model("gpt-4.1-nano".to_string()) .build(); - // Create the LM instance via the builder + // Create the LM instance via the async builder let lm = LM::builder() .config(config) .api_key(env::var("OPENAI_API_KEY")?.into()) - .build(); + .build() + .await; // Configure the global LM and adapter configure(lm, ChatAdapter::default()); @@ -149,10 +151,11 @@ async fn main() -> Result<(), anyhow::Error> { let lm = LM::builder() .config(config) .api_key(env::var("OPENAI_API_KEY")?.into()) - .build(); + .build() + .await; configure(lm.clone(), ChatAdapter::default()); - // Create a questin-answering signature instance + // Create a question-answering signature instance let signature = QA::new(); // Create a predictor let predictor = Predict::new(signature); @@ -233,7 +236,8 @@ async fn main() -> anyhow::Result<()> { let lm: LM = LM::builder() .config(config) .api_key(env::var("OPENAI_API_KEY")?.into()) - .build(); + .build() + .await; configure(lm, ChatAdapter::default()); // Create the module instance