Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions deploy/helm/rockbot/templates/blazor/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ spec:
{{- include "rockbot.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: blazor
spec:
# Match the shared-data fsGroup so files the agent wrote (group rw via its own fsGroup)
# are readable here. The PVC is co-mounted read-only solely to serve attachments.
securityContext:
fsGroup: {{ .Values.shared.fsGroup }}
containers:
- name: blazor
image: {{ .Values.blazor.image.repository }}:{{ .Values.blazor.image.tag }}
Expand Down Expand Up @@ -55,6 +59,10 @@ spec:
secretKeyRef:
name: {{ include "rockbot.secretName" . }}
key: RabbitMq__Password
# Where the shared attachments directory is mounted — the /attachments endpoint
# resolves AgentReply.Attachments file references under ${ROCKBOT_SHARED_PATH}/attachments.
- name: ROCKBOT_SHARED_PATH
value: "/rockbot/shared"
{{- if .Values.workiq.enabled }}
# WorkIQ device-code flow needs the same Entra app registration
# values the agent uses. Cache itself never lands on the Blazor pod.
Expand Down Expand Up @@ -98,3 +106,12 @@ spec:
periodSeconds: 5
resources:
{{- toYaml .Values.blazor.resources | nindent 12 }}
volumeMounts:
# Read-only: Blazor only serves attachment bytes; the agent owns writes.
- name: shared-data
mountPath: /rockbot/shared
readOnly: true
volumes:
- name: shared-data
persistentVolumeClaim:
claimName: {{ include "rockbot.sharedPvcName" . }}
16 changes: 13 additions & 3 deletions design/client-rendering-capabilities.md
Original file line number Diff line number Diff line change
Expand Up @@ -351,9 +351,19 @@ would need re-negotiation when the surface changes.

These are deliberately out of scope for v1:

- **`AgentReply.Attachments`** — needed for `ImageAttachment` capability to do
anything end-to-end. Real refactor (new field, proxy support for binary
payloads in each platform). Defer until a proxy that can use it ships.
- **`AgentReply.Attachments`** — ✅ **implemented** (issue #416, first slice). A new
`IReadOnlyList<AgentAttachment>?` field on `AgentReply` carries out-of-band binaries as
shared-PVC **path references** (`{ mime, path, fileName? }`), reusing the MCP-attachment
scheme — bytes never ride the bus. Producing side: an `attach_image` LLM tool
(`AttachmentReplyTools`) validates a model-named file under the shared attachments dir and
stages it in a session-keyed `ReplyAttachmentBuffer`; `UserMessageHandler` drains the buffer
onto the final reply (the **user-message path** only in v1). Rendering side: Blazor
co-mounts the shared PVC read-only and serves bytes from a minimal `/attachments` endpoint,
rendering images as native `<img>` **outside** the markdown sanitizer (which strips `<img>`
and `data:` URLs); the CLI prints a placeholder line per attachment.
Still deferred: scheduled-task / subagent / A2A producing paths (the buffer generalizes
trivially), SVG→PNG rasterization, chat-platform proxies (none exist yet), and attachments in
conversation-history replay (replies are ephemeral; history stores text).
- **Platform-native UI emission** — `DiscordEmbed` / `SlackBlockKit` /
`TeamsAdaptiveCard` bits are reserved in the enum but no tooling produces
them. Adding them is opportunistic upgrade work in each proxy (e.g., a
Expand Down
153 changes: 153 additions & 0 deletions src/RockBot.Agent/AttachmentReplyTools.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
using System.ComponentModel;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Logging;
using RockBot.Agent.McpBridge.Attachments;
using RockBot.Host;
using RockBot.UserProxy;

namespace RockBot.Agent;

/// <summary>
/// Per-session LLM tool that lets the agent attach an image file to its final reply. The file
/// must already exist under the shared attachments directory — produced by a script or MCP
/// tool — so the agent never handles bytes itself (the "Nothing trusts the LLM" rule). The
/// tool validates containment and existence, infers the MIME type from the extension when not
/// given, and stages an <see cref="AgentAttachment"/> in the <see cref="ReplyAttachmentBuffer"/>
/// keyed to this session. The reply-publishing path drains the buffer onto the final reply.
/// </summary>
public sealed class AttachmentReplyTools
{
private readonly IAttachmentStorage _storage;
private readonly ReplyAttachmentBuffer _buffer;
private readonly string _sessionId;
private readonly ILogger _logger;

public AttachmentReplyTools(
IAttachmentStorage storage,
ReplyAttachmentBuffer buffer,
string sessionId,
ILogger logger)
{
_storage = storage;
_buffer = buffer;
_sessionId = sessionId;
_logger = logger;

Tools = [AIFunctionFactory.Create(AttachImage)];
}

public IList<AITool> Tools { get; }

[Description(
"Attach an image file to your reply so the user sees it rendered inline (e.g. a chart, " +
"diagram, or screenshot). The file must already exist in the shared attachments directory " +
"— first have a script or tool write it there, then call this with the filename. Do NOT " +
"embed images as markdown or data URLs; those are stripped. Only call this once the file " +
"exists. The image is shown to clients that can render it; others see a short placeholder line.")]
public string AttachImage(
[Description("Filename of the image under the shared attachments directory (e.g. 'chart.png'). " +
"Must be a file that already exists there.")] string path,
[Description("Optional MIME type (e.g. 'image/png'). Inferred from the file extension when omitted.")] string? mime = null,
[Description("Optional friendly display name shown to the user. Defaults to the filename.")] string? fileName = null)
{
_logger.LogInformation("Tool call: AttachImage(path={Path}, mime={Mime})", path, mime);

if (string.IsNullOrWhiteSpace(path))
return "Could not attach: no path provided. Pass the filename of an image already written to the shared attachments directory.";

string resolved;
try
{
resolved = ResolveUnderBase(path);
}
catch (UnauthorizedAccessException)
{
return $"Could not attach '{path}': it is outside the shared attachments directory. " +
"Only files under the shared attachments directory can be attached.";
}

if (!File.Exists(resolved))
return $"Could not attach '{path}': no such file in the shared attachments directory. " +
"Write the image there first, then attach it.";

var resolvedMime = string.IsNullOrWhiteSpace(mime) ? GuessMime(resolved) : mime.Trim();
var leaf = Path.GetFileName(path);
var displayName = string.IsNullOrWhiteSpace(fileName) ? leaf : fileName.Trim();

_buffer.Add(_sessionId, new AgentAttachment
{
Mime = resolvedMime,
Path = leaf,
FileName = displayName
});

return $"Attached '{displayName}' ({resolvedMime}). It will be included with your reply.";
}

/// <summary>
/// Resolves <paramref name="path"/> to an absolute path under <see cref="IAttachmentStorage.BasePath"/>,
/// throwing <see cref="UnauthorizedAccessException"/> if it escapes the base. Mirrors
/// <c>AttachmentStorage.ResolveReadPath</c> so model-controlled input cannot reach arbitrary
/// filesystem locations.
/// </summary>
private string ResolveUnderBase(string path)
{
var fullBase = Path.GetFullPath(_storage.BasePath);

string candidate;
if (Path.IsPathRooted(path))
{
candidate = Path.GetFullPath(path);
}
else
{
// The shared-volume convention refers to files as `<subdir>/<file>`; the model may
// write `attachments/foo.png` even though BasePath already ends in `/attachments`.
// Strip the redundant leaf so it resolves to a single layer.
candidate = Path.GetFullPath(Path.Combine(fullBase, StripRedundantBaseLeaf(fullBase, path)));
}

var baseWithSep = fullBase.EndsWith(Path.DirectorySeparatorChar)
? fullBase
: fullBase + Path.DirectorySeparatorChar;
if (!candidate.StartsWith(baseWithSep, StringComparison.OrdinalIgnoreCase)
&& !candidate.Equals(fullBase, StringComparison.OrdinalIgnoreCase))
{
throw new UnauthorizedAccessException(
$"Attachment path '{path}' is outside the shared attachments directory '{_storage.BasePath}'.");
}

return candidate;
}

private static string StripRedundantBaseLeaf(string basePath, string relativePath)
{
var leaf = Path.GetFileName(basePath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
if (string.IsNullOrEmpty(leaf)) return relativePath;

ReadOnlySpan<char> span = relativePath;
if (span.StartsWith(leaf, StringComparison.OrdinalIgnoreCase)
&& span.Length > leaf.Length
&& (span[leaf.Length] == '/' || span[leaf.Length] == '\\'))
{
return relativePath[(leaf.Length + 1)..];
}
return relativePath;
}

private static string GuessMime(string fileName)
{
var ext = Path.GetExtension(fileName).ToLowerInvariant();
return ext switch
{
".png" => "image/png",
".jpg" or ".jpeg" => "image/jpeg",
".gif" => "image/gif",
".webp" => "image/webp",
".svg" => "image/svg+xml",
".bmp" => "image/bmp",
".pdf" => "application/pdf",
_ => "application/octet-stream"
};
}
}
6 changes: 6 additions & 0 deletions src/RockBot.Agent/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,12 @@ async Task<IChatClient> BuildClientForTierAsync(LlmTierConfig config, string tie
// without rebuilding the image. LlmPricing__ConfigPath in the ConfigMap overrides the default.
builder.Services.Configure<LlmPricingOptions>(builder.Configuration.GetSection("LlmPricing"));

// Shared attachments storage — the filesystem facade over ${ROCKBOT_SHARED_PATH}/attachments.
// Used by the attach_image reply tool to validate model-named files before referencing them.
// McpBridgeService keeps its own Lazy instance; this registration limits blast radius.
builder.Services.AddSingleton<RockBot.Agent.McpBridge.Attachments.IAttachmentStorage,
RockBot.Agent.McpBridge.Attachments.AttachmentStorage>();

// MCP bridge (replaces external RockBot.Tools.Mcp.Bridge process)
builder.Services.Configure<McpBridgeOptions>(builder.Configuration.GetSection("McpBridge"));
builder.Services.AddSingleton(new McpArgGuardRegistration(
Expand Down
21 changes: 20 additions & 1 deletion src/RockBot.Agent/UserMessageHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ internal sealed class UserMessageHandler(
ISessionTracker sessionTracker,
SessionStartTracker sessionStartTracker,
SessionClientCapabilityStore clientCapabilityStore,
ReplyAttachmentBuffer attachmentBuffer,
McpBridge.Attachments.IAttachmentStorage attachmentStorage,
SessionOriginStore originStore,
IOptions<AgentProfileOptions> profileOptions,
IWipTracker wipTracker,
Expand Down Expand Up @@ -184,6 +186,10 @@ await conversationMemory.AddTurnAsync(
var sessionNamespace = $"session/{message.SessionId}";
var sessionWorkingMemoryTools = new WorkingMemoryTools(workingMemory, sessionNamespace, logger);

// Per-session attachment tool — attach_image stages files for this session's final reply.
var attachmentReplyTools = new AttachmentReplyTools(
attachmentStorage, attachmentBuffer, message.SessionId, logger);

// Per-session skill tools with usage tracking
var sessionSkillTools = new SkillTools(skillStore, llmClient, logger, message.SessionId, skillUsageStore);

Expand All @@ -193,6 +199,7 @@ await conversationMemory.AddTurnAsync(

var allTools = memoryTools.Tools
.Concat(sessionWorkingMemoryTools.Tools)
.Concat(attachmentReplyTools.Tools)
.Concat(sessionSkillTools.Tools)
.Concat(rulesTools.Tools)
.Concat(toolGuideTools.Tools)
Expand Down Expand Up @@ -691,12 +698,24 @@ private async Task PublishReplyAsync(
string content, string replyTo, string? correlationId,
string sessionId, bool isFinal, CancellationToken ct)
{
// Only final replies carry attachments — drain the per-session buffer so files staged
// by attach_image ride out with the answer and aren't replayed on a later turn.
// Progress/non-final replies never carry attachments.
IReadOnlyList<AgentAttachment>? attachments = null;
if (isFinal)
{
var drained = attachmentBuffer.Drain(sessionId);
if (drained.Count > 0)
attachments = drained;
}

var reply = new AgentReply
{
Content = content,
SessionId = sessionId,
AgentName = agentNameHolder.DisplayName ?? agent.Name,
IsFinal = isFinal
IsFinal = isFinal,
Attachments = attachments
};
var envelope = reply.ToEnvelope<AgentReply>(source: agent.Name, correlationId: correlationId);
await publisher.PublishAsync(replyTo, envelope, ct);
Expand Down
56 changes: 56 additions & 0 deletions src/RockBot.Host/ReplyAttachmentBuffer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
using RockBot.UserProxy;

namespace RockBot.Host;

/// <summary>
/// Session-keyed buffer of <see cref="AgentAttachment"/> the agent has staged for the next
/// final reply. The <c>attach_image</c> LLM tool calls <see cref="Add"/> during a turn; the
/// reply-publishing path calls <see cref="Drain"/> when it emits the final reply, moving the
/// staged attachments onto <see cref="AgentReply.Attachments"/> and clearing the buffer so
/// they aren't replayed on a later turn.
/// <para>
/// Mirrors <see cref="SessionClientCapabilityStore"/> in shape and lifetime: a process-wide
/// singleton holding short-lived per-session metadata, guarded by a single lock. Registered
/// as a singleton in the agent host.
/// </para>
/// </summary>
public sealed class ReplyAttachmentBuffer
{
private readonly Dictionary<string, List<AgentAttachment>> _bySession
= new(StringComparer.OrdinalIgnoreCase);
private readonly Lock _lock = new();

/// <summary>
/// Stage <paramref name="attachment"/> for <paramref name="sessionId"/>'s next final reply.
/// </summary>
public void Add(string sessionId, AgentAttachment attachment)
{
lock (_lock)
{
if (!_bySession.TryGetValue(sessionId, out var list))
_bySession[sessionId] = list = new List<AgentAttachment>();
list.Add(attachment);
}
}

/// <summary>
/// Returns and clears the attachments staged for <paramref name="sessionId"/>. Returns an
/// empty list when nothing is staged. Drains so a later reply in the same session starts clean.
/// </summary>
public IReadOnlyList<AgentAttachment> Drain(string sessionId)
{
lock (_lock)
{
if (_bySession.Remove(sessionId, out var list))
return list;
return [];
}
}

/// <summary>Drop any staged attachments for <paramref name="sessionId"/> without returning them.</summary>
public void Clear(string sessionId)
{
lock (_lock)
_bySession.Remove(sessionId);
}
}
1 change: 1 addition & 0 deletions src/RockBot.Host/ServiceCollectionExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public static IServiceCollection AddRockBotHost(
services.AddScoped<AgentContextBuilder>();
services.AddSingleton<SessionStartTracker>();
services.AddSingleton<SessionClientCapabilityStore>();
services.AddSingleton<ReplyAttachmentBuffer>();
services.AddSingleton<SessionOriginStore>();
services.AddSingleton<IUserActivityMonitor, UserActivityMonitor>();
services.AddSingleton<ISessionTracker, SessionBackgroundTaskTracker>();
Expand Down
26 changes: 26 additions & 0 deletions src/RockBot.UserProxy.Abstractions/AgentAttachment.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
namespace RockBot.UserProxy;

/// <summary>
/// A binary payload attached to an <see cref="AgentReply"/>, carried as a path reference
/// rather than inline bytes. <see cref="Path"/> names a file the agent has produced (via a
/// script or MCP tool) under the shared attachments directory
/// (<c>${ROCKBOT_SHARED_PATH}/attachments</c>) — the same convention the MCP attachment
/// gateway speaks. Bytes never ride the bus: a capable frontend (e.g. Blazor) co-mounts the
/// shared volume and serves the file from its own endpoint; non-image frontends render a
/// placeholder.
/// </summary>
public sealed record AgentAttachment
{
/// <summary>MIME type of the attachment (e.g. <c>image/png</c>).</summary>
public required string Mime { get; init; }

/// <summary>
/// Bare filename or relative path under the shared attachments directory (e.g.
/// <c>chart.png</c>). Never an absolute filesystem path on the wire — the receiving
/// frontend resolves it under its own attachments base with a containment check.
/// </summary>
public required string Path { get; init; }

/// <summary>Optional human-friendly display name; falls back to the leaf of <see cref="Path"/>.</summary>
public string? FileName { get; init; }
}
9 changes: 9 additions & 0 deletions src/RockBot.UserProxy.Abstractions/AgentReply.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,13 @@ public sealed record AgentReply
/// or arrived in a different client.
/// </summary>
public ReplyOrigin? Origin { get; init; }

/// <summary>
/// Out-of-band binary payloads (images, etc.) the agent produced for this reply, carried
/// as shared-volume path references rather than inline bytes. Only set on final replies —
/// progress/non-final replies never carry attachments. Null when the reply is text-only.
/// Capable frontends render these natively (Blazor serves them from an HTTP endpoint);
/// others print a placeholder line. Serializes for free via the camelCase STJ path.
/// </summary>
public IReadOnlyList<AgentAttachment>? Attachments { get; init; }
}
Loading
Loading