diff --git a/README.md b/README.md index 0047471c..12107e26 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ It’s designed to be **fast**, **simple**, and **extremely slim** — with a ve - `0 → N` wake-up from **Kafka lag** via the companion **SlimFaas Kafka** service, - `N → M` scaling powered by PromQL, - internal metrics store, debug endpoints, and scale-to-zero out of the box. -- temporary **Data Binary** endpoints to ingest and stage files (from tiny to very large) with TTL-friendly storage — perfect for caching & agentic workflows. +- temporary **Data Files** endpoints to ingest and stage binaries (from tiny to very large) with TTL-friendly storage — perfect for caching & agentic workflows. +- temporary **Data Sets** endpoints (`/data/sets`) to store small, Redis-like KV payloads (cache, JSON state, flags) with optional TTL — replicated through the cluster via a robust consensus layer. > **Looking for MCP integration?** > Check out **[SlimFaas MCP](https://slimfaas.dev/mcp)** — the companion runtime that converts *any* OpenAPI definition into MCP-ready tools on the fly. @@ -102,9 +103,13 @@ It’s designed to be **fast**, **simple**, and **extremely slim** — with a ve - Synchronously send events to **every replica** of selected functions. - No additional event bus required — ideal for cluster-local fan-out, cache invalidation, configuration refresh, etc. -### 📁 Data & Files (real-time ingestion + ephemeral caching) +### 🧰 Data Files & Data Sets (real-time ingestion + ephemeral caching + robust KV) -SlimFaas includes **Data Files** endpoints to **stream, store, and serve temporary files** — from tiny payloads to *very large* binaries. +SlimFaas includes two complementary “data” APIs: + +#### 📁 Data Files (temporary binary artifacts) + +**Data Files** endpoints are designed to **stream, store, and serve temporary files** — from tiny payloads to *very large* binaries. Ideal for **agentic workflows** and **real-time ingestion**: upload once, get an `id`, then let tools/functions consume it when they’re ready. - Stream-first uploads (without buffering in memory or disk) @@ -112,6 +117,15 @@ Ideal for **agentic workflows** and **real-time ingestion**: upload once, get an - **Ephemeral caching** for intermediate artifacts - **TTL-based** lifecycle (auto-expiration) +#### 🧠 Data Sets (Redis-like KV for small state) + +**Data Sets** endpoints provide a **small, Redis-like KV store** (raw bytes) replicated across the SlimFaas cluster. + +- Stream-first uploads +- Store **anything small**: JSON, strings, flags, lightweight cache entries +- Optional `ttl` in **milliseconds** (auto-expiration) +- Hard limit: **1 MiB per value** + ### 🧠 “Mind Changer” (Status & Wake-up API) - Built-in REST APIs to: @@ -144,16 +158,17 @@ Check out: - [Get Started](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/get-started.md) – Learn how to deploy SlimFaas on Kubernetes or Docker Compose. - Scaling - - [Autoscaling](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/autoscaling.md) – Deep-dive into `0 → N` / `N → M` autoscaling, PromQL triggers, metrics scraping, and debug endpoints. - - [Kafka Connector](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/kafka.md) – Use Kafka topic lag to wake functions from `0 → N` and keep workers alive while messages are still flowing. - - [Planet Saver](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/planet-saver.md) – See how to start and monitor replicas from a JavaScript frontend. + - [Autoscaling](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/autoscaling.md) – Deep-dive into `0 → N` / `N → M` autoscaling, PromQL triggers, metrics scraping, and debug endpoints. + - [Kafka Connector](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/kafka.md) – Use Kafka topic lag to wake functions from `0 → N` and keep workers alive while messages are still flowing. + - [Planet Saver](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/planet-saver.md) – See how to start and monitor replicas from a JavaScript frontend. - Functions & Workloads - - [Functions](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/functions.md) – See how to call functions synchronously or asynchronously. - - [Events](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/events.md) – Explore how to use internal synchronous publish/subscribe events. - - [Jobs](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/jobs.md) – Learn how to define and run one-off jobs. - - [OpenTelemetry](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/opentelemetry.md) – Enable distributed tracing, metrics, and logs with OpenTelemetry integration. + - [Functions](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/functions.md) – See how to call functions synchronously or asynchronously. + - [Events](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/events.md) – Explore how to use internal synchronous publish/subscribe events. + - [Jobs](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/jobs.md) – Learn how to define and run one-off jobs. + - [OpenTelemetry](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/opentelemetry.md) – Enable distributed tracing, metrics, and logs with OpenTelemetry integration. - Data & Files - - [Data Files](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/data-files.md) - Understand how to ingest, store, and serve temporary binary artifacts. + - [Data Files](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/data-files.md) - Understand how to ingest, store, and serve temporary binary artifacts. + - [Data Sets](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/data-sets.md) - Store small, Redis-like KV payloads (cache, JSON state, flags) replicated across the cluster, with optional TTL. - [How It Works](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/how-it-works.md) – Dive into SlimFaas’s architecture and design. - [MCP](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/mcp.md) – Discover how to convert *any* OpenAPI definition into MCP-ready tools on the fly. diff --git a/demo/deployment-functions.yml b/demo/deployment-functions.yml index ef4b9d74..a78bab6f 100644 --- a/demo/deployment-functions.yml +++ b/demo/deployment-functions.yml @@ -42,13 +42,13 @@ spec: "ScaleUp": { "StabilizationWindowSeconds": 0, "Policies": [ - { "Type": "Pods", "Value": 1, "PeriodSeconds": 30 } + { "Type": "Pods", "Value": 1, "PeriodSeconds": 10 } ] }, "ScaleDown": { "StabilizationWindowSeconds": 20, "Policies": [ - { "Type": "Pods", "Value": 1, "PeriodSeconds": 30 } + { "Type": "Pods", "Value": 1, "PeriodSeconds": 10 } ] } } diff --git a/documentation/data-sets.md b/documentation/data-sets.md new file mode 100644 index 00000000..52f3df26 --- /dev/null +++ b/documentation/data-sets.md @@ -0,0 +1,156 @@ +# Data Sets API + +SlimFaas provides a **Redis-like, cluster-consistent key/value store** through the `/data/sets` endpoints. + +Use it to store **small values** (cache entries, JSON state, flags, checkpoints) with a robust replication protocol (SlimData + Raft). + +- **Max payload size:** 1 MiB per entry +- **TTL unit:** milliseconds (`ttl` query parameter) + +> Not for large file storage. For large binaries (PDF, ZIP, audio, PPTX, ...), use **Data Files**: https://slimfaas.dev/data-files + +--- + +## How it works (high-level) + +```mermaid +flowchart LR + C[Client] -->|HTTP| API[SlimFaas /data/sets] + API -->|SetAsync / GetAsync / DeleteAsync| DB[SlimData] + DB -->|Raft replication| N1[(Node A)] + DB -->|Raft replication| N2[(Node B)] + DB -->|Raft replication| N3[(Node C)] + DB --> TTL[TTL metadata] + TTL --> EXP[Auto-expire] +``` + +--- + +## Endpoints + +Base path: `/data/sets` + +| Method | Path | Purpose | +|---|---|---| +| `POST` | `/data/sets?id={id?}&ttl={ttl_ms?}` | Create or overwrite a value | +| `GET` | `/data/sets/{id}` | Read a value | +| `GET` | `/data/sets` | List entries (IDs + expiration) | +| `DELETE` | `/data/sets/{id}` | Delete a value | + +### IDs + +- IDs are validated server-side (`IdValidator.IsSafeId`). +- If `id` is omitted (or empty), SlimFaas generates one (`Guid.NewGuid().ToString("N")`) and returns it. + +### TTL (milliseconds) + +- `ttl` is optional and expressed in **milliseconds**. +- When provided, the entry auto-expires after the TTL. + +Examples: +- `ttl=60000` → 1 minute +- `ttl=600000` → 10 minutes + +--- + +## Create / overwrite + +`POST /data/sets?id={id?}&ttl={ttl_ms?}` + +- Body is stored as raw bytes. +- Payload larger than 1 MiB returns **413 Payload Too Large**. + +Examples: + +Store JSON with a fixed id: +```bash +curl -X POST "http:///data/sets?id=my-usecase/session-123/state" \ + -H "Content-Type: application/json" \ + --data-binary '{"step":"route","chosen":"kb_rag","confidence":0.92}' +``` + +Store a string: +```bash +curl -X POST "http:///data/sets?id=my-usecase/session-123/flag" \ + -H "Content-Type: text/plain" \ + --data-binary "ready" +``` + +Let SlimFaas generate the id: +```bash +ID=$(curl -s -X POST "http:///data/sets" --data-binary "hello") +echo "created id=$ID" +``` + +Store with TTL (10 minutes = 600000 ms): +```bash +curl -X POST "http:///data/sets?id=my-usecase/session-123/state&ttl=600000" \ + --data-binary "temporary" +``` + +--- + +## Read + +`GET /data/sets/{id}` + +- Returns raw bytes as `application/octet-stream`. +- `404 Not Found` when missing or expired. + +Examples: +```bash +curl -L "http:///data/sets/my-usecase/session-123/state" -o value.bin +``` + +If you stored JSON: +```bash +curl -s "http:///data/sets/my-usecase/session-123/state" | jq . +``` + +--- + +## List + +`GET /data/sets` + +Returns a JSON array: +```json +[ + { "id": "abc", "expireAtUtcTicks": -1 }, + { "id": "xyz", "expireAtUtcTicks": 638720123456789012 } +] +``` + +- `expireAtUtcTicks` is UTC DateTime ticks (100 ns units). +- `-1` means “no expiration”. + +--- + +## Delete + +`DELETE /data/sets/{id}` + +- Returns **204 No Content**. + +Example: +```bash +curl -X DELETE "http:///data/sets/my-usecase/session-123/state" +``` + +--- + +## Visibility & security + +`/data/sets` is protected by the same data visibility policy used by other data endpoints (`DataVisibilityEndpointFilter`). + +`appsettings.json`: +```json +{ + "Data": { + "DefaultVisibility": "Private" + } +} +``` + +Env override: +- `Data:DefaultVisibility` → `Data__DefaultVisibility` diff --git a/documentation/home.md b/documentation/home.md index d4229789..dd83563a 100644 --- a/documentation/home.md +++ b/documentation/home.md @@ -19,7 +19,8 @@ It’s designed to be **fast**, **simple**, and **extremely slim** — with a ve - `0 → N` wake-up from **Kafka lag** via the companion **SlimFaas Kafka** service, - `N → M` scaling powered by PromQL, - internal metrics store, debug endpoints, and scale-to-zero out of the box. -- temporary **Data Binary** endpoints to ingest and stage files (from tiny to very large) with TTL-friendly storage — perfect for caching & agentic workflows. +- temporary **Data Sets** endpoints (Redis-like KV) to store small blobs/JSON with **TTL (milliseconds)** — perfect for cache & agentic state. +- temporary **Data Files** endpoints to ingest and stage files (from tiny to very large) with TTL-friendly storage — perfect for caching & agentic workflows. > **Looking for MCP integration?** > Check out **[SlimFaas MCP](https://slimfaas.dev/mcp)** — the companion runtime that converts *any* OpenAPI definition into MCP-ready tools on the fly. @@ -93,15 +94,12 @@ SlimFaas puts autoscaling at the center of the design: - Synchronously broadcast events to **every replica** of selected functions. - No external event bus required — perfect for simple fan-out, cache invalidation, or configuration refresh scenarios. -### 📁 Data & Files (real-time ingestion + ephemeral caching) +### 🧺 Data Sets & 📁 Data Files (ephemeral state + real-time ingestion) -SlimFaas includes **Data Files** endpoints to **stream, store, and serve temporary files** — from tiny payloads to *very large* binaries. -Ideal for **agentic workflows** and **real-time ingestion**: upload once, get an `id`, then let tools/functions consume it when they’re ready. +SlimFaas includes two complementary data APIs: -- Stream-first uploads (without buffering in memory or disk) -- **Agentic-ready** attachments & multi-step flows -- **Ephemeral caching** for intermediate artifacts -- **TTL-based** lifecycle (auto-expiration) +- **Data Sets** (`/data/sets`): a Redis-like, cluster-consistent **KV store** for small payloads (cache, JSON state, flags, checkpoints) with optional **TTL in milliseconds** and a **1 MiB** payload limit. +- **Data Files** (`/data/files`): stream-first endpoints to ingest, store, and serve temporary files — from tiny payloads to *very large* binaries — ideal for **agentic workflows** and **real-time ingestion** (upload once, get an `id`, then let tools/functions consume it when they’re ready). ### 🧠 “Mind Changer” (Status & Wake-up API) @@ -143,6 +141,7 @@ Dive into the documentation: - [Jobs](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/jobs.md) – Learn how to define and run one-off jobs. - [OpenTelemetry](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/opentelemetry.md) – Enable distributed tracing, metrics, and logs with OpenTelemetry integration. - Data & Files + - [Data Sets](https://slimfaas.dev/data-sets) - Store small blobs/JSON in a Redis-like KV store with TTL (milliseconds). - [Data Files](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/data-files.md) - Understand how to ingest, store, and serve temporary binary artifacts. - [How It Works](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/how-it-works.md) – Dive into SlimFaas’s architecture and design. - [MCP](https://github.com/SlimPlanet/SlimFaas/blob/main/documentation/mcp.md) – Discover how to convert *any* OpenAPI definition into MCP-ready tools on the fly. diff --git a/src/SlimFaas/Data/DataSetRoutes.cs b/src/SlimFaas/Data/DataSetRoutes.cs index ded7be3b..12a8000c 100644 --- a/src/SlimFaas/Data/DataSetRoutes.cs +++ b/src/SlimFaas/Data/DataSetRoutes.cs @@ -17,13 +17,15 @@ public static class DataSetRoutes private static string DataKey(string id) => $"{SetPrefix}{id}"; private static string TtlKey(string key) => key + SlimDataInterpreter.TimeToLivePostfix; - public static IEndpointRouteBuilder MapDataSetRoutes(this IEndpointRouteBuilder app) + public static IEndpointRouteBuilder MapDataSetRoutes(this IEndpointRouteBuilder endpoints) { - app.MapPost("/data/sets", Handlers.PostAsync); - app.MapGet("/data/sets/{id}", Handlers.GetAsync); - app.MapGet("/data/sets", Handlers.ListAsync); - app.MapDelete("/data/sets/{id}", Handlers.DeleteAsync); - return app; + var group = endpoints.MapGroup("/data/sets") + .AddEndpointFilter(); + group.MapPost("", Handlers.PostAsync); + group.MapGet("/{id}", Handlers.GetAsync); + group.MapGet("", Handlers.ListAsync); + group.MapDelete("/{id}", Handlers.DeleteAsync); + return endpoints; } diff --git a/src/SlimFaas/Database/SlimDataService.cs b/src/SlimFaas/Database/SlimDataService.cs index f06e1018..e2e646cc 100644 --- a/src/SlimFaas/Database/SlimDataService.cs +++ b/src/SlimFaas/Database/SlimDataService.cs @@ -42,17 +42,6 @@ public SlimDataService( maxWaitPerTick: TimeSpan.FromSeconds(5) ); - var tiersLlp = new[] - { - new RateTier(20, TimeSpan.FromMilliseconds(250)), - new RateTier(300, TimeSpan.FromMilliseconds(500)), - }; - var tiersLcb = new[] - { - new RateTier(50, TimeSpan.FromMilliseconds(150)), - new RateTier(500, TimeSpan.FromMilliseconds(400)), - }; - _batcher.RegisterKind( kind: "llp", batchHandler: BatchHandlerAsync, diff --git a/src/SlimFaas/Program.cs b/src/SlimFaas/Program.cs index b10ce999..a0c7042f 100644 --- a/src/SlimFaas/Program.cs +++ b/src/SlimFaas/Program.cs @@ -488,7 +488,7 @@ }); //app.MapDataHashsetRoutes(); -//app.MapDataSetRoutes(); +app.MapDataSetRoutes(); app.MapDataFileRoutes(); app.MapDebugRoutes(); diff --git a/src/SlimFaas/appsettings.json b/src/SlimFaas/appsettings.json index fbd44a81..d0d7f7eb 100644 --- a/src/SlimFaas/appsettings.json +++ b/src/SlimFaas/appsettings.json @@ -28,7 +28,7 @@ "Endpoint": "", "ServiceName": "", "EnableConsoleExporter": false, - "ExcludedUrls": ["/health", "/metrics", "/ready", "/SlimData", "/status-functions", "/cluster-consensus"] + "ExcludedUrls": ["/health", "/metrics", "/ready", "/SlimData", "/status-functions", "/cluster-consensus", "/wake-function"] }, "Data": { "DefaultVisibility": "Private" diff --git a/src/SlimFaasSite/src/components/NavBar.tsx b/src/SlimFaasSite/src/components/NavBar.tsx index 018fe323..8c2f6ae3 100644 --- a/src/SlimFaasSite/src/components/NavBar.tsx +++ b/src/SlimFaasSite/src/components/NavBar.tsx @@ -6,23 +6,33 @@ const Navbar: React.FC = () => { const [isMenuOpen, setIsMenuOpen] = useState(false); const [isScaleOpen, setIsScaleOpen] = useState(false); const [isFunctionsOpen, setIsFunctionsOpen] = useState(false); + const [isDataOpen, setIsDataOpen] = useState(false); const toggleMenu = () => setIsMenuOpen((prev) => !prev); const toggleScale = () => { setIsScaleOpen((prev) => !prev); setIsFunctionsOpen(false); + setIsDataOpen(false); }; const toggleFunctions = () => { setIsFunctionsOpen((prev) => !prev); setIsScaleOpen(false); + setIsDataOpen(false); + }; + + const toggleData = () => { + setIsDataOpen((prev) => !prev); + setIsScaleOpen(false); + setIsFunctionsOpen(false); }; const closeAll = () => { setIsMenuOpen(false); setIsScaleOpen(false); setIsFunctionsOpen(false); + setIsDataOpen(false); }; return ( @@ -190,21 +200,54 @@ const Navbar: React.FC = () => { -
  • - + + +
    • +
  • - + How it works
    • diff --git a/src/SlimFaasSite/src/pages/data-sets.tsx b/src/SlimFaasSite/src/pages/data-sets.tsx new file mode 100644 index 00000000..b6beedef --- /dev/null +++ b/src/SlimFaasSite/src/pages/data-sets.tsx @@ -0,0 +1,33 @@ +import React from 'react'; +import Layout from '../components/Layout'; +import { GetStaticProps } from 'next'; +import { fetchMarkdownFile, MarkdownData, MarkdownMetadata } from '../lib/github'; +import { renderMarkdownWithHighlight } from '@/lib/markdown'; + +export interface DocPageProps { + contentHtml: string; + metadata: MarkdownMetadata; +} + + +export const getStaticProps: GetStaticProps = async () => { + const markdownFile: MarkdownData = await fetchMarkdownFile(`documentation/data-sets.md`); + const contentHtml = await renderMarkdownWithHighlight(markdownFile.contentHtml); + + return { + props: { + contentHtml: contentHtml, + metadata: markdownFile.metadata || {}, + }, + }; +}; + +const PlanetSaver = ({ contentHtml }: DocPageProps) => ( + +
    +
    +
    + +); + +export default PlanetSaver;