Merge pull request #78 from tinyhumansai/graph-view-and-ingest-api

Add a graph view, document and URL intake, and a namespace convention

Author: senamakel

Cargo.lock

@@ -43,6 +43,10 @@ crates/
 │                       them
 ├── tinymemory-sources/ memory-source contracts and readers — local folders
 │                       always, GitHub/RSS/web pages behind `network`
+├── tinymemory-documents/ document and URL intake: sniff a format, convert it
+│                       to markdown, and write it into whichever engine is
+│                       bound. The URL half is behind `network` and reuses the
+│                       source readers' SSRF guard rather than growing a second
 ├── tinymemory-tinycortex/  the TinyCortex engine seen through the contract
 ├── tinymemory-remote/  native HTTP dialects for Supermemory, Mem0, and Cognee
 ├── tinymemory-conformance/ the behavioural suite every driver must pass
@@ -79,6 +83,8 @@ composition — no storage engine, no HTTP stack, no native library.
 | `sync` | `tinymemory::sync` — the Composio normalisers |
 | `sources` | `tinymemory::sources` — source contracts and local readers |
 | `sources-network` | `sources`, plus the GitHub/RSS/web-page readers |
+| `documents` | `tinymemory::documents` — document intake and markdown conversion |
+| `documents-network` | `documents`, plus the URL fetch path |
 | `conformance` | `tinymemory::conformance` — the driver contract suite |
 | `memory-git` | git-backed diff snapshots (implies `tinycortex`; links libgit2) |
 | `contacts` | the macOS address-book seeding path (implies `core`) |

README.md

@@ -70,6 +70,13 @@
 //! - [`traits`]: the [`traits::Memory`] storage-backend trait.
 //! - [`chunks`]: the persisted chunk model ([`chunks::Chunk`], [`chunks::Metadata`],
 //!   [`chunks::SourceRef`], …) and the deterministic [`chunks::chunk_id`].
+//! - [`graph`]: the bounded graph-view model ([`graph::GraphView`],
+//!   [`graph::GraphViewQuery`], [`graph::GraphNode`], [`graph::GraphEdge`]) —
+//!   the graph counterpart of [`tree`], and what
+//!   [`provider::MemoryGraph::graph_view`] returns.
+//! - [`namespace`]: the `<section>:<scope>` namespace convention
+//!   ([`namespace::Namespace`], [`namespace::MemorySection`]) and its
+//!   validator.
 //! - [`tree`]: the markdown summary-tree node model ([`tree::TreeNode`],
 //!   [`tree::NodeLevel`], [`tree::TreeStatus`], …).
 //! - [`tool_memory`]: tool-scoped rule contracts ([`tool_memory::ToolMemoryRule`], …).
@@ -99,7 +106,8 @@ pub mod host;
 // point: a second definition would need a conversion at the module seam that
 // nothing type-checks.
 pub use tinymemory_bus::{
-    capabilities, chunks, error, goals, health, recall, tool_memory, tree, types, version, wire,
+    capabilities, chunks, error, goals, graph, health, namespace, recall, tool_memory, tree, types,
+    version, wire,
 };
 /// The mandatory-family composition: wrap any [`traits::Memory`] backend as a
 /// complete [`provider::MemoryProvider`].

crates/tinymemory-api/src/lib.rs

@@ -10,12 +10,26 @@
 //! extraction models, hotness decay curves, and snapshot retention are driver
 //! concerns and appear in none of these signatures.
 
+use std::collections::BTreeSet;
+
 use async_trait::async_trait;
 
 use crate::error::MemoryError;
+use crate::graph::{GraphEdge, GraphNode, GraphView, GraphViewQuery};
 use crate::provider::types::{DiffReport, EntityHit, SnapshotRef};
 use crate::types::{GraphRelationRecord, MemoryKvRecord};
 
+/// How many edges the default [`MemoryGraph::graph_view`] traversal scans per
+/// predicate when it has to resolve *inbound* edges.
+///
+/// [`MemoryGraph::relations`] filters by subject and predicate but not by
+/// object, so inbound expansion has no indexed form in this contract and the
+/// default traversal falls back to a bounded scan. The bound exists so a graph
+/// larger than memory cannot be pulled into a view; hitting it sets
+/// [`GraphView::truncated`]. A driver whose store indexes the object column
+/// should override `graph_view` and skip this path entirely.
+pub const INBOUND_SCAN_LIMIT: usize = 4_096;
+
 /// The entity index: who and what the stored memory is about.
 #[async_trait]
 pub trait MemoryEntities: Send + Sync {
@@ -140,6 +154,238 @@ pub trait MemoryGraph: Send + Sync {
     /// [`MemoryError::Invalid`] for a malformed edge, otherwise backend
     /// failures.
     async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError>;
+
+    /// Assemble a bounded, renderable slice of the graph.
+    ///
+    /// This is to [`Self::relations`] what
+    /// [`crate::provider::MemoryTree::drill_down`] is to a raw node read: one
+    /// call returns a node *together with its surroundings*, already joined
+    /// into a node set and an edge set, so navigating a graph is a sequence of
+    /// view calls rather than a client-side reassembly that every caller would
+    /// write differently.
+    ///
+    /// # The default implementation
+    ///
+    /// Provided, not required: it breadth-first expands
+    /// [`GraphViewQuery::seeds`] using [`Self::relations`] alone, so every
+    /// existing driver gains a graph view without writing one, and a driver
+    /// that advertises no [`crate::capabilities::Capability::Graph`] family
+    /// still surfaces the same [`MemoryError::Unsupported`] its `relations`
+    /// returns.
+    ///
+    /// It costs one `relations` call per node visited, including one final
+    /// round at the outermost hop that adds no nodes and exists only to close
+    /// edges *between* nodes already in the view — without it the outer ring
+    /// renders as a star rather than as the graph it is. A driver with a native
+    /// multi-hop traversal should override this and use it.
+    ///
+    /// Inbound expansion has no indexed form here — `relations` cannot filter
+    /// by object — so [`crate::graph::GraphDirection::In`] and
+    /// [`crate::graph::GraphDirection::Both`] fall back to a scan capped at
+    /// [`INBOUND_SCAN_LIMIT`] per predicate.
+    ///
+    /// # Errors
+    ///
+    /// Whatever [`Self::relations`] returns. Bounds are never an error: a
+    /// traversal that hits one returns the partial view with
+    /// [`GraphView::truncated`] set.
+    async fn graph_view(&self, query: &GraphViewQuery) -> Result<GraphView, MemoryError> {
+        let namespace = query.namespace.as_deref();
+        let mut view = GraphView {
+            namespace: query.namespace.clone(),
+            seeds: query.seeds.clone(),
+            ..GraphView::default()
+        };
+
+        // Each predicate needs its own call: `relations` takes one, not a set.
+        // An empty filter becomes the single unfiltered call rather than a
+        // special case further down.
+        let predicates: Vec<Option<&str>> = if query.predicates.is_empty() {
+            vec![None]
+        } else {
+            query.predicates.iter().map(|p| Some(p.as_str())).collect()
+        };
+
+        // Nodes reached but never expanded, either because a bound was hit or
+        // because they sit one hop past the requested depth. A set, not a
+        // counter: the same boundary node is commonly reached from several
+        // directions, and counting it twice would overstate what is left.
+        let mut unexpanded: BTreeSet<String> = BTreeSet::new();
+
+        // Unseeded: an overview, not a traversal. One bounded scan of the
+        // slice, and the node set is whatever the returned edges touch.
+        if query.seeds.is_empty() {
+            for predicate in &predicates {
+                // One past the ceiling: a call that asked for exactly
+                // `max_edges` and got them cannot tell a full slice from a
+                // truncated one.
+                let records = self
+                    .relations(
+                        namespace,
+                        None,
+                        *predicate,
+                        query.max_edges.saturating_add(1),
+                    )
+                    .await?;
+                for record in records {
+                    push_view_edge(&mut view, record, &mut unexpanded, query, 0);
+                }
+            }
+            view.stats.frontier_remaining = unexpanded.len();
+            view.recompute_stats();
+            return Ok(view);
+        }
+
+        // Inbound edges are resolved from one scan per predicate rather than
+        // one per node: the scan is the expensive part, and repeating it for
+        // every node visited would multiply it by `max_nodes`.
+        let mut inbound: Vec<GraphRelationRecord> = Vec::new();
+        if query.direction.follows_in() {
+            for predicate in &predicates {
+                let records = self
+                    .relations(namespace, None, *predicate, INBOUND_SCAN_LIMIT)
+                    .await?;
+                if records.len() >= INBOUND_SCAN_LIMIT {
+                    view.truncated = true;
+                }
+                inbound.extend(records);
+            }
+        }
+
+        let mut frontier: Vec<String> = Vec::new();
+        for seed in &query.seeds {
+            if view.nodes.iter().any(|n| &n.id == seed) {
+                continue;
+            }
+            if view.nodes.len() >= query.max_nodes {
+                unexpanded.insert(seed.clone());
+                view.truncated = true;
+                continue;
+            }
+            view.nodes.push(GraphNode::bare(seed.clone(), 0));
+            frontier.push(seed.clone());
+        }
+
+        for hop in 0..=query.depth {
+            if frontier.is_empty() {
+                break;
+            }
+            let mut next: Vec<String> = Vec::new();
+            for node_id in &frontier {
+                let mut incident: Vec<GraphRelationRecord> = Vec::new();
+                if query.direction.follows_out() {
+                    for predicate in &predicates {
+                        incident.extend(
+                            self.relations(
+                                namespace,
+                                Some(node_id),
+                                *predicate,
+                                query.max_edges.saturating_add(1),
+                            )
+                            .await?,
+                        );
+                    }
+                }
+                if query.direction.follows_in() {
+                    incident.extend(
+                        inbound
+                            .iter()
+                            .filter(|record| &record.object == node_id)
+                            .cloned(),
+                    );
+                }
+
+                for record in incident {
+                    if !query.accepts_predicate(&record.predicate) {
+                        continue;
+                    }
+                    let other = if &record.subject == node_id {
+                        record.object.clone()
+                    } else {
+                        record.subject.clone()
+                    };
+                    let known = view.nodes.iter().any(|n| n.id == other);
+                    if !known {
+                        // An edge to a node the view will not hold would
+                        // dangle, so it is dropped either way — but *why* it
+                        // was dropped matters. Reaching the requested depth is
+                        // the caller getting what they asked for; hitting the
+                        // node ceiling is not, and only the second makes the
+                        // view truncated. Conflating them would set the flag on
+                        // every finite traversal of a connected graph and leave
+                        // it saying nothing.
+                        if hop >= query.depth {
+                            unexpanded.insert(other);
+                            continue;
+                        }
+                        if view.nodes.len() >= query.max_nodes {
+                            unexpanded.insert(other);
+                            view.truncated = true;
+                            continue;
+                        }
+                        view.nodes.push(GraphNode::bare(other.clone(), hop + 1));
+                        next.push(other);
+                    }
+                    push_view_edge(&mut view, record, &mut unexpanded, query, hop);
+                }
+            }
+            frontier = next;
+        }
+
+        view.stats.frontier_remaining = unexpanded.len();
+        view.recompute_stats();
+        Ok(view)
+    }
+}
+
+/// Add one relation to a view, deduplicating by triple and honouring
+/// [`GraphViewQuery::max_edges`].
+///
+/// A separate function rather than a closure so the borrow of `view` ends
+/// between calls, which the traversal above needs while it is also pushing
+/// nodes.
+fn push_view_edge(
+    view: &mut GraphView,
+    record: GraphRelationRecord,
+    unexpanded: &mut BTreeSet<String>,
+    query: &GraphViewQuery,
+    depth: u32,
+) {
+    if !query.accepts_predicate(&record.predicate) {
+        return;
+    }
+    let triple = (
+        record.subject.clone(),
+        record.predicate.clone(),
+        record.object.clone(),
+    );
+    if view
+        .edges
+        .iter()
+        .any(|e| e.key() == (&triple.0, &triple.1, &triple.2))
+    {
+        return;
+    }
+    if view.edges.len() >= query.max_edges {
+        unexpanded.insert(triple.0);
+        unexpanded.insert(triple.2);
+        view.truncated = true;
+        return;
+    }
+    // The unseeded overview derives its node set from the edges it found; the
+    // seeded traversal has already placed both endpoints.
+    for id in [triple.0.clone(), triple.2.clone()] {
+        if view.nodes.iter().any(|n| n.id == id) {
+            continue;
+        }
+        if view.nodes.len() >= query.max_nodes {
+            unexpanded.insert(id);
+            view.truncated = true;
+            return;
+        }
+        view.nodes.push(GraphNode::bare(id, depth));
+    }
+    view.edges.push(GraphEdge::from(record));
 }
 
 /// Snapshot capture and change computation over synced sources.

crates/tinymemory-api/src/provider/knowledge.rs

@@ -78,7 +78,7 @@ pub use chunks::{ChunkDetail, ChunkEmbedding, ChunkQuery, MemoryChunks};
 pub use content::{MemoryDocuments, MemoryIngest, MemoryTree};
 pub use driver::MemoryProvider;
 pub use episodic::{ConversationSegment, EpisodicTurn, MemoryEpisodic};
-pub use knowledge::{MemoryDiff, MemoryEntities, MemoryGraph};
+pub use knowledge::{MemoryDiff, MemoryEntities, MemoryGraph, INBOUND_SCAN_LIMIT};
 pub use mandatory::{MemoryCore, MemoryPortability, MemoryRecall};
 pub use people::{
     AddressBookSeedOutcome, MemoryPeople, PersonHandle, PersonInteraction, PersonRecord, PersonRef,