diff --git a/guides/configuration.md b/guides/configuration.md index 3ac204ac..8b66fcb8 100644 --- a/guides/configuration.md +++ b/guides/configuration.md @@ -150,6 +150,78 @@ config :req_llm, {:ok, response} = ReqLLM.stream_text(model, messages, finch_name: MyApp.CustomFinch) ``` +## Req Plugins + +ReqLLM builds a `Req.Request` struct internally for each operation. The `:req_plugins` option lets you inject custom middleware into this request before it's sent, enabling custom headers, logging, metrics, retries, or any other `Req` step. + +Each plugin is a function `(Req.Request.t() -> Req.Request.t())` applied in order. If a plugin raises, the operation returns `{:error, ...}` without sending the request. + +### Adding Custom Headers + +```elixir +ReqLLM.generate_text("openai:gpt-4o", "Hello", + req_plugins: [ + fn req -> Req.Request.put_header(req, "x-request-id", UUID.uuid4()) end + ] +) +``` + +### Composing Multiple Plugins + +Plugins are applied left to right: + +```elixir +auth_plugin = fn req -> + Req.Request.put_header(req, "x-custom-auth", "Bearer #{my_token()}") +end + +logging_plugin = fn req -> + Logger.info("Sending request to #{req.url}") + req +end + +ReqLLM.generate_text("openai:gpt-4o", messages, + req_plugins: [auth_plugin, logging_plugin] +) +``` + +### Reusable Plugin Modules + +For plugins shared across your application, define a module: + +```elixir +defmodule MyApp.ReqPlugins do + def request_id(req) do + Req.Request.put_header(req, "x-request-id", UUID.uuid4()) + end + + def telemetry(req) do + Req.Request.register_options(req, [:telemetry_metadata]) + Req.Request.append_request_steps(req, telemetry: &emit_start/1) + end + + defp emit_start(req) do + :telemetry.execute([:my_app, :llm, :request], %{}, %{url: req.url}) + req + end +end + +ReqLLM.generate_text("openai:gpt-4o", messages, + req_plugins: [&MyApp.ReqPlugins.request_id/1, &MyApp.ReqPlugins.telemetry/1] +) +``` + +### Supported Operations + +The `:req_plugins` option works with all non-streaming operations: + +- `ReqLLM.generate_text/3` +- `ReqLLM.generate_object/4` +- `ReqLLM.embed/3` (single and batch) +- `ReqLLM.generate_image/3` + +Streaming operations (`stream_text/3`) build their requests through a different pipeline and do not currently support `:req_plugins`. + ## API Key Configuration Keys are loaded with clear precedence: per-request → in-memory → app config → env vars → .env files. diff --git a/lib/req_llm/embedding.ex b/lib/req_llm/embedding.ex index 80dd1c86..8eea2360 100644 --- a/lib/req_llm/embedding.ex +++ b/lib/req_llm/embedding.ex @@ -178,6 +178,7 @@ defmodule ReqLLM.Embedding do :ok <- validate_input(text), {:ok, provider_module} <- ReqLLM.provider(model.provider), {:ok, request} <- provider_module.prepare_request(:embedding, model, text, opts), + {:ok, request} <- ReqLLM.Provider.Defaults.apply_req_plugins(request, opts), {:ok, %Req.Response{status: status, body: decoded_response}} when status in 200..299 <- Req.request(request) do extract_single_embedding(decoded_response) @@ -200,6 +201,7 @@ defmodule ReqLLM.Embedding do :ok <- validate_input(texts), {:ok, provider_module} <- ReqLLM.provider(model.provider), {:ok, request} <- provider_module.prepare_request(:embedding, model, texts, opts), + {:ok, request} <- ReqLLM.Provider.Defaults.apply_req_plugins(request, opts), {:ok, %Req.Response{status: status, body: decoded_response}} when status in 200..299 <- Req.request(request) do extract_multiple_embeddings(decoded_response) diff --git a/lib/req_llm/generation.ex b/lib/req_llm/generation.ex index e92fe778..fcdb1882 100644 --- a/lib/req_llm/generation.ex +++ b/lib/req_llm/generation.ex @@ -75,6 +75,7 @@ defmodule ReqLLM.Generation do with {:ok, model} <- ReqLLM.model(model_spec), {:ok, provider_module} <- ReqLLM.provider(model.provider), {:ok, request} <- provider_module.prepare_request(:chat, model, messages, opts), + {:ok, request} <- ReqLLM.Provider.Defaults.apply_req_plugins(request, opts), {:ok, %Req.Response{status: status, body: decoded_response}} when status in 200..299 <- Req.request(request) do {:ok, decoded_response} @@ -240,6 +241,7 @@ defmodule ReqLLM.Generation do opts_with_schema = Keyword.put(opts, :compiled_schema, compiled_schema), {:ok, request} <- provider_module.prepare_request(:object, model, messages, opts_with_schema), + {:ok, request} <- ReqLLM.Provider.Defaults.apply_req_plugins(request, opts), {:ok, %Req.Response{status: status, body: decoded_response}} when status in 200..299 <- Req.request(request) do # For models with json.strict = false, coerce response types to match schema diff --git a/lib/req_llm/images.ex b/lib/req_llm/images.ex index f0abb8a0..fbbc74a2 100644 --- a/lib/req_llm/images.ex +++ b/lib/req_llm/images.ex @@ -112,6 +112,7 @@ defmodule ReqLLM.Images do {:ok, provider_module} <- ReqLLM.provider(model.provider), {:ok, request} <- provider_module.prepare_request(:image, model, prompt_or_messages, opts), + {:ok, request} <- ReqLLM.Provider.Defaults.apply_req_plugins(request, opts), {:ok, %Req.Response{status: status, body: response}} when status in 200..299 <- Req.request(request) do {:ok, response} diff --git a/lib/req_llm/provider/defaults.ex b/lib/req_llm/provider/defaults.ex index f8910a75..3033acac 100644 --- a/lib/req_llm/provider/defaults.ex +++ b/lib/req_llm/provider/defaults.ex @@ -349,12 +349,49 @@ defmodule ReqLLM.Provider.Defaults do :operation, :receive_timeout, :max_retries, - :req_http_options + :req_http_options, + :req_plugins ] Keyword.drop(opts, internal_keys) end + @doc """ + Applies user-provided Req plugins to a built request. + + Each plugin is a function `(Req.Request.t() -> Req.Request.t())` applied in order. + A crashing plugin returns `{:error, ...}` rather than propagating the exception. + """ + @spec apply_req_plugins(Req.Request.t(), keyword()) :: + {:ok, Req.Request.t()} | {:error, Exception.t()} + def apply_req_plugins(request, opts) do + plugins = Keyword.get(opts, :req_plugins, []) + + result = + Enum.reduce_while(plugins, request, fn plugin, req -> + try do + {:cont, plugin.(req)} + rescue + error -> + {:halt, {:error, error, plugin}} + end + end) + + case result do + {:error, error, plugin} -> + {:error, + ReqLLM.Error.Unknown.Unknown.exception( + error: + RuntimeError.exception( + "Req plugin #{inspect(plugin)} failed: #{Exception.message(error)}" + ) + )} + + %Req.Request{} = req -> + {:ok, req} + end + end + @spec default_attach(module(), Req.Request.t(), term(), keyword()) :: Req.Request.t() def default_attach(provider_mod, %Req.Request{} = request, model_input, user_opts) do {:ok, %LLMDB.Model{} = model} = ReqLLM.model(model_input) @@ -405,6 +442,7 @@ defmodule ReqLLM.Provider.Defaults do :tools, :tool_choice, :req_http_options, + :req_plugins, :stream, :frequency_penalty, :system_prompt, diff --git a/lib/req_llm/provider/options.ex b/lib/req_llm/provider/options.ex index 0e4aca92..350bb5c3 100644 --- a/lib/req_llm/provider/options.ex +++ b/lib/req_llm/provider/options.ex @@ -171,6 +171,7 @@ defmodule ReqLLM.Provider.Options do :on_unsupported, :fixture, :req_http_options, + :req_plugins, :compiled_schema, :operation, :text, diff --git a/test/req_llm/req_plugins_test.exs b/test/req_llm/req_plugins_test.exs new file mode 100644 index 00000000..f4c433ed --- /dev/null +++ b/test/req_llm/req_plugins_test.exs @@ -0,0 +1,283 @@ +defmodule ReqLLM.ReqPluginsTest do + use ExUnit.Case, async: true + + alias ReqLLM.{Generation, Embedding, Response} + + @chat_response %{ + "id" => "cmpl_plugins_test", + "model" => "gpt-4o-mini-2024-07-18", + "choices" => [ + %{ + "message" => %{"role" => "assistant", "content" => "Hello from plugins test!"} + } + ], + "usage" => %{"prompt_tokens" => 10, "completion_tokens" => 8, "total_tokens" => 18} + } + + @embedding_response %{ + "object" => "list", + "data" => [%{"object" => "embedding", "index" => 0, "embedding" => [0.1, 0.2, 0.3]}], + "model" => "text-embedding-3-small", + "usage" => %{"prompt_tokens" => 5, "total_tokens" => 5} + } + + @object_response %{ + "id" => "cmpl_object_test", + "model" => "gpt-4o-mini-2024-07-18", + "choices" => [ + %{ + "message" => %{ + "role" => "assistant", + "content" => nil, + "tool_calls" => [ + %{ + "id" => "call_1", + "type" => "function", + "function" => %{ + "name" => "structured_output", + "arguments" => ~s({"name":"Alice","age":30}) + } + } + ] + } + } + ], + "usage" => %{"prompt_tokens" => 20, "completion_tokens" => 15, "total_tokens" => 35} + } + + describe "generate_text/3 with :req_plugins" do + test "single plugin adds a custom header" do + test_pid = self() + + Req.Test.stub(ReqLLM.ReqPluginsTest.SingleHeader, fn conn -> + send(test_pid, {:headers, conn.req_headers}) + Req.Test.json(conn, @chat_response) + end) + + plugin = fn req -> + Req.Request.put_header(req, "x-custom-plugin", "test-value") + end + + {:ok, response} = + Generation.generate_text( + "openai:gpt-4o-mini", + "Hello", + req_plugins: [plugin], + req_http_options: [plug: {Req.Test, ReqLLM.ReqPluginsTest.SingleHeader}] + ) + + assert %Response{} = response + assert_receive {:headers, headers} + assert List.keyfind(headers, "x-custom-plugin", 0) == {"x-custom-plugin", "test-value"} + end + + test "multiple plugins applied in order" do + test_pid = self() + + Req.Test.stub(ReqLLM.ReqPluginsTest.MultiPlugins, fn conn -> + send(test_pid, {:headers, conn.req_headers}) + Req.Test.json(conn, @chat_response) + end) + + plugin_a = fn req -> + Req.Request.put_header(req, "x-order", "first") + end + + plugin_b = fn req -> + Req.Request.put_header(req, "x-order", "second") + end + + {:ok, _response} = + Generation.generate_text( + "openai:gpt-4o-mini", + "Hello", + req_plugins: [plugin_a, plugin_b], + req_http_options: [plug: {Req.Test, ReqLLM.ReqPluginsTest.MultiPlugins}] + ) + + assert_receive {:headers, headers} + assert List.keyfind(headers, "x-order", 0) == {"x-order", "second"} + end + + test "empty list is a no-op" do + Req.Test.stub(ReqLLM.ReqPluginsTest.EmptyPlugins, fn conn -> + Req.Test.json(conn, @chat_response) + end) + + {:ok, response} = + Generation.generate_text( + "openai:gpt-4o-mini", + "Hello", + req_plugins: [], + req_http_options: [plug: {Req.Test, ReqLLM.ReqPluginsTest.EmptyPlugins}] + ) + + assert %Response{} = response + end + + test "crashing plugin returns {:error, ...} with message" do + crashing_plugin = fn _req -> + raise "plugin exploded" + end + + {:error, error} = + Generation.generate_text( + "openai:gpt-4o-mini", + "Hello", + req_plugins: [crashing_plugin], + req_http_options: [plug: {Req.Test, __MODULE__}] + ) + + assert Exception.message(error) =~ "plugin exploded" + end + + test "first plugin crash stops pipeline" do + test_pid = self() + + crashing = fn _req -> raise "boom" end + + second = fn req -> + send(test_pid, :second_called) + req + end + + {:error, _} = + Generation.generate_text( + "openai:gpt-4o-mini", + "Hello", + req_plugins: [crashing, second], + req_http_options: [plug: {Req.Test, __MODULE__}] + ) + + refute_receive :second_called + end + end + + describe "generate_object/4 with :req_plugins" do + test "plugins applied to object generation requests" do + test_pid = self() + + Req.Test.stub(ReqLLM.ReqPluginsTest.ObjectPlugin, fn conn -> + send(test_pid, {:headers, conn.req_headers}) + Req.Test.json(conn, @object_response) + end) + + plugin = fn req -> + Req.Request.put_header(req, "x-object-plugin", "active") + end + + schema = [ + name: [type: :string], + age: [type: :integer] + ] + + {:ok, response} = + Generation.generate_object( + "openai:gpt-4o-mini", + "Generate a person", + schema, + req_plugins: [plugin], + req_http_options: [plug: {Req.Test, ReqLLM.ReqPluginsTest.ObjectPlugin}] + ) + + assert %Response{} = response + assert_receive {:headers, headers} + assert List.keyfind(headers, "x-object-plugin", 0) == {"x-object-plugin", "active"} + end + end + + describe "embed/3 with :req_plugins" do + test "plugins applied to single text embedding" do + test_pid = self() + + Req.Test.stub(ReqLLM.ReqPluginsTest.EmbedPlugin, fn conn -> + send(test_pid, {:headers, conn.req_headers}) + Req.Test.json(conn, @embedding_response) + end) + + plugin = fn req -> + Req.Request.put_header(req, "x-embed-plugin", "yes") + end + + {:ok, embedding} = + Embedding.embed( + "openai:text-embedding-3-small", + "Hello world", + req_plugins: [plugin], + req_http_options: [plug: {Req.Test, ReqLLM.ReqPluginsTest.EmbedPlugin}] + ) + + assert is_list(embedding) + assert_receive {:headers, headers} + assert List.keyfind(headers, "x-embed-plugin", 0) == {"x-embed-plugin", "yes"} + end + + test "plugins applied to batch embedding" do + test_pid = self() + + batch_response = %{ + "object" => "list", + "data" => [ + %{"object" => "embedding", "index" => 0, "embedding" => [0.1, 0.2]}, + %{"object" => "embedding", "index" => 1, "embedding" => [0.3, 0.4]} + ], + "model" => "text-embedding-3-small", + "usage" => %{"prompt_tokens" => 10, "total_tokens" => 10} + } + + Req.Test.stub(ReqLLM.ReqPluginsTest.BatchEmbedPlugin, fn conn -> + send(test_pid, {:headers, conn.req_headers}) + Req.Test.json(conn, batch_response) + end) + + plugin = fn req -> + Req.Request.put_header(req, "x-batch-embed", "true") + end + + {:ok, embeddings} = + Embedding.embed( + "openai:text-embedding-3-small", + ["Hello", "World"], + req_plugins: [plugin], + req_http_options: [plug: {Req.Test, ReqLLM.ReqPluginsTest.BatchEmbedPlugin}] + ) + + assert length(embeddings) == 2 + assert_receive {:headers, headers} + assert List.keyfind(headers, "x-batch-embed", 0) == {"x-batch-embed", "true"} + end + end + + describe "apply_req_plugins/2 unit" do + test "returns {:ok, request} when no plugins" do + req = Req.new(url: "https://example.com") + assert {:ok, ^req} = ReqLLM.Provider.Defaults.apply_req_plugins(req, []) + end + + test "returns {:ok, request} with empty plugin list" do + req = Req.new(url: "https://example.com") + assert {:ok, ^req} = ReqLLM.Provider.Defaults.apply_req_plugins(req, req_plugins: []) + end + + test "applies plugin transformation" do + req = Req.new(url: "https://example.com") + + plugin = fn r -> Req.Request.put_header(r, "x-test", "value") end + + assert {:ok, result} = + ReqLLM.Provider.Defaults.apply_req_plugins(req, req_plugins: [plugin]) + + assert Req.Request.get_header(result, "x-test") == ["value"] + end + + test "returns error on plugin crash" do + req = Req.new(url: "https://example.com") + plugin = fn _r -> raise "kaboom" end + + assert {:error, error} = + ReqLLM.Provider.Defaults.apply_req_plugins(req, req_plugins: [plugin]) + + assert Exception.message(error) =~ "kaboom" + end + end +end