Run Composio normalisation and storage on a driver that is not TinyCortex (#18 §B3)

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
-members = [".", "api", "core", "adapters/tinycortex", "adapters/remote", "conformance"]
-default-members = [".", "api", "core", "adapters/tinycortex", "adapters/remote", "conformance"]
+# `sync` is the engine-neutral Composio normalisers (issue #18 §B3).
+members = [".", "api", "core", "sync", "adapters/tinycortex", "adapters/remote", "conformance"]
+default-members = [".", "api", "core", "sync", "adapters/tinycortex", "adapters/remote", "conformance"]
 # `vendor/` holds engine submodules (tinycortex, tinybus, tinyagents), each of
 # which is its own workspace with its own lockfile. Same exclusion
 # `vendor/tinycortex` uses for its own nested vendor directory.
@@ -79,6 +80,9 @@ tinymemory-api = { path = "api", features = ["test-support"] }
 # integration tests. A dev-dependency only: the facade must not carry a test
 # harness into a consumer's dependency graph.
 tinymemory-conformance = { path = "conformance" }
+# The extracted Composio normalisers, for the integration test that runs them
+# against a driver that is not TinyCortex (issue #18 §B3).
+tinymemory-sync = { path = "sync" }
 
 [features]
 default = []

core/Cargo.toml

@@ -16,6 +16,11 @@ readme = "../README.md"
 # The contract. `tinymemory-core` implements and consumes it; the host seam
 # traits (config, event sink, embeddings, chat) live in `tinymemory_api::host`.
 tinymemory-api = { path = "../api" }
+# Composio payload normalisers, extracted out of the engine (issue #18 §B3).
+# They were reached through `tinycortex` until now, which meant a host binding a
+# different engine could not have Composio sync despite none of this code
+# caring which engine is bound.
+tinymemory-sync = { path = "../sync" }
 tinymemory = { path = ".." }
 
 # The default embedded engine. `store/`, `tree/` and `sync/` drive it directly;

core/src/sync/composio/providers/clickup/mod.rs

@@ -17,7 +17,7 @@
 // The payload normalisers moved to tinycortex (they are pure Value
 // transforms, i.e. driver-side). Aliased under the old module name so
 // every `normalization::extract_*` call site below stays unchanged.
-use crate::engine::backend::sync::composio::providers::normalize::clickup as normalization;
+use tinymemory_sync::clickup as normalization;
 mod provider;
 #[cfg(test)]
 mod tests;

core/src/sync/composio/providers/github/mod.rs

@@ -16,7 +16,7 @@
 // The payload normalisers moved to tinycortex (they are pure Value
 // transforms, i.e. driver-side). Aliased under the old module name so
 // every `normalization::extract_*` call site below stays unchanged.
-use crate::engine::backend::sync::composio::providers::normalize::github as normalization;
+use tinymemory_sync::github as normalization;
 mod provider;
 #[cfg(test)]
 mod tests;

core/src/sync/composio/providers/gmail/mod.rs

@@ -1,7 +1,7 @@
 // The Gmail post-processor moved to tinycortex (a pure Value transform, i.e.
 // driver-side). Aliased under the old module name so the single call site in
 // `provider.rs` stays unchanged.
-use crate::engine::backend::sync::composio::providers::normalize::gmail_post_process as post_process;
+use tinymemory_sync::gmail_post_process as post_process;
 mod provider;
 #[cfg(test)]
 mod tests;

core/src/sync/composio/providers/helpers.rs

@@ -1,11 +1,11 @@
 //! Shared helpers for Composio provider implementations.
 //!
 //! `pick_str` used to live here. It is a provider payload normaliser, so it
-//! moved to `crate::engine::backend::sync::composio::providers::normalize::helpers`
+//! moved to `tinymemory_sync::helpers`
 //! and is re-exported from this module's parent. The helpers that remain are
 //! request-building rather than normalisation, and stay host-side.
 
-use crate::engine::backend::sync::composio::providers::normalize::helpers::pick_str;
+use tinymemory_sync::helpers::pick_str;
 
 /// Shallow-merge an `extra` JSON object into a (mutable) action-args
 /// object. Only object-typed extras are merged; non-object `extra`

core/src/sync/composio/providers/linear/mod.rs

@@ -6,7 +6,7 @@
 // The payload normalisers moved to tinycortex (they are pure Value
 // transforms, i.e. driver-side). Aliased under the old module name so
 // every `normalization::extract_*` call site below stays unchanged.
-use crate::engine::backend::sync::composio::providers::normalize::linear as normalization;
+use tinymemory_sync::linear as normalization;
 mod provider;
 #[cfg(test)]
 mod tests;

core/src/sync/composio/providers/mod.rs

@@ -281,11 +281,11 @@ pub(crate) use helpers::{first_array_str, merge_extra};
 // re-exported here so the ~40 in-tree call sites keep resolving unchanged.
 // Note this is deliberately NOT `providers::common::pick_str`, which coerces
 // numbers to strings — see the doc comments on both definitions.
-pub(crate) use crate::engine::backend::sync::composio::providers::normalize::helpers::pick_str;
 pub use registry::{
     all_providers, get_provider, init_default_providers, register_provider, ProviderArc,
 };
 pub use scope_lookup::{curated_scope_for, toolkit_has_scope};
+pub(crate) use tinymemory_sync::helpers::pick_str;
 pub use tool_scope::{classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope};
 pub use traits::{resolve_sync_interval_secs, sync_interval_env_var, ComposioProvider};
 pub use types::{

core/src/sync/composio/providers/notion/mod.rs

@@ -1,7 +1,7 @@
 // The payload normalisers moved to tinycortex (they are pure Value
 // transforms, i.e. driver-side). Aliased under the old module name so
 // every `normalization::extract_*` call site below stays unchanged.
-use crate::engine::backend::sync::composio::providers::normalize::notion as normalization;
+use tinymemory_sync::notion as normalization;
 mod provider;
 #[cfg(test)]
 mod tests;

core/src/sync/composio/providers/slack/mod.rs

@@ -10,7 +10,7 @@
 // driver-side). Re-exported under the old module name — `pub`, not a plain
 // `use`, because `tests/raw_coverage/memory_threads_raw_coverage_e2e.rs`
 // imports this path directly.
-pub use crate::engine::backend::sync::composio::providers::normalize::slack_post_process as post_process;
+pub use tinymemory_sync::slack_post_process as post_process;
 pub mod types;
 
 mod provider;

sync/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "tinymemory-sync"
+publish = false
+version = "0.1.0"
+edition = "2021"
+rust-version = "1.96"
+license = "MIT"
+repository = "https://github.com/tinyhumansai/tinymemory"
+description = "Engine-neutral Composio payload normalisers for TinyMemory"
+
+# The whole dependency list, and it is the point of the crate. These are pure
+# `Value -> Value` transforms: no engine, no storage, no network, no async
+# runtime. A dependency added here should have to argue for itself against that
+# sentence (issue #18 §B3).
+[dependencies]
+serde_json = "1"
+# Two logging facades, neither an implementation, both carried over from the
+# engine layout this crate was extracted from: `gmail_post_process` traces
+# through `tracing`, `slack_post_process` through `log`. Preserved rather than
+# unified, because §B3 is a *move* and swapping a facade would change where a
+# host's log lines surface — a behaviour change hiding inside a relocation.
+# Worth reconciling in its own change.
+tracing = "0.1"
+log = "0.4"
+# RFC 2822/3339 date handling for Gmail `Date:` headers.
+#
+# `clock` is on, and it is the one place this crate is not a pure function of
+# its input: `format_email_local_time` renders in `chrono::Local`, so it reads
+# the host's timezone. That is deliberate upstream — the agent presents local
+# times without doing UTC arithmetic, and the raw UTC field is preserved
+# alongside — but it means "pure `Value -> Value`" is true of every normaliser
+# here except that one. Better said out loud than discovered by someone whose
+# output moved when they changed TZ.
+chrono = { version = "0.4", features = ["clock"] }
+
+[lints.rust]
+unsafe_code = "forbid"
+missing_docs = "warn"
+unreachable_pub = "warn"
+
+[lints.clippy]
+all = { level = "warn", priority = -1 }
+unwrap_used = "warn"
+expect_used = "warn"
+panic = "warn"

sync/src/clickup.rs

@@ -0,0 +1,133 @@
+//! ClickUp host normalization helpers — result extraction, task-title extraction,
+//! and time utilities.
+//!
+//! ClickUp's REST API (and therefore Composio's wrapping of it) returns
+//! task lists in a small handful of shapes depending on which endpoint
+//! is called. The functions here walk the union of common shapes so the
+//! provider doesn't have to branch per Composio envelope variant.
+
+use serde_json::Value;
+
+use super::helpers::pick_str;
+
+/// Walk the Composio response envelope for ClickUp task list results.
+///
+/// ClickUp's "filtered team tasks" endpoint returns `{ "tasks": [...] }`
+/// at the top level; Composio re-wraps the upstream payload under
+/// `data` or `data.data` depending on the action. We probe each shape
+/// in order and return the first array we find.
+pub fn extract_tasks(data: &Value) -> Vec<Value> {
+    let candidates = [
+        data.pointer("/data/tasks"),
+        data.pointer("/tasks"),
+        data.pointer("/data/data/tasks"),
+        data.pointer("/data/results"),
+        data.pointer("/results"),
+        data.pointer("/data/items"),
+        data.pointer("/items"),
+    ];
+    for cand in candidates.into_iter().flatten() {
+        if let Some(arr) = cand.as_array() {
+            return arr.clone();
+        }
+    }
+    Vec::new()
+}
+
+/// Extract a human-readable title from a ClickUp task object.
+///
+/// ClickUp tasks store the name at `name` (or `data.name` after Composio
+/// envelope wrapping). When the name is missing we fall back to the
+/// task ID so chunks remain identifiable.
+pub fn extract_task_name(task: &Value) -> Option<String> {
+    pick_str(task, &["name", "data.name", "title", "data.title"])
+}
+
+/// Extract a stable cursor timestamp (milliseconds since epoch as a
+/// string) from a ClickUp task object.
+///
+/// The ClickUp API returns `date_updated` as a stringified epoch ms
+/// (e.g. `"1733412345678"`); we keep it as a string so lexicographic
+/// comparison against the stored cursor remains valid as long as the
+/// length doesn't change (it won't until year 33658).
+pub fn extract_task_updated(task: &Value) -> Option<String> {
+    pick_str(
+        task,
+        &[
+            "date_updated",
+            "data.date_updated",
+            "updated_at",
+            "data.updated_at",
+            "dateUpdated",
+            "data.dateUpdated",
+        ],
+    )
+}
+
+/// Current wall-clock time in milliseconds since the UNIX epoch.
+pub fn now_ms() -> u64 {
+    use std::time::{SystemTime, UNIX_EPOCH};
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_millis() as u64)
+        .unwrap_or(0)
+}
+
+/// Extract the authorized user's numeric ID from the
+/// `CLICKUP_GET_AUTHORIZED_USER` response.
+///
+/// Composio wraps the upstream `{"user": {"id": …}}` shape; this walker
+/// is defensive against both raw and wrapped payloads. Returns the ID
+/// as a string because `CLICKUP_GET_FILTERED_TEAM_TASKS` accepts the
+/// `assignees` filter as a string array.
+pub fn extract_user_id(data: &Value) -> Option<String> {
+    let candidates = [
+        data.pointer("/user/id"),
+        data.pointer("/data/user/id"),
+        data.pointer("/id"),
+        data.pointer("/data/id"),
+    ];
+    for cand in candidates.into_iter().flatten() {
+        if let Some(n) = cand.as_u64() {
+            return Some(n.to_string());
+        }
+        if let Some(n) = cand.as_i64() {
+            return Some(n.to_string());
+        }
+        if let Some(s) = cand.as_str() {
+            let trimmed = s.trim();
+            if !trimmed.is_empty() {
+                return Some(trimmed.to_string());
+            }
+        }
+    }
+    None
+}
+
+/// Extract a list of workspace (team) IDs from the
+/// `CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES` response.
+///
+/// ClickUp returns `{"teams": [{"id": "...", "name": "..."}, …]}`. We
+/// keep the IDs as strings — `CLICKUP_GET_FILTERED_TEAM_TASKS` requires
+/// a `team_id` (string) argument.
+pub fn extract_workspace_ids(data: &Value) -> Vec<String> {
+    let candidates = [
+        data.pointer("/teams"),
+        data.pointer("/data/teams"),
+        data.pointer("/workspaces"),
+        data.pointer("/data/workspaces"),
+    ];
+    for cand in candidates.into_iter().flatten() {
+        if let Some(arr) = cand.as_array() {
+            return arr
+                .iter()
+                .filter_map(|t| pick_str(t, &["id", "team_id", "workspace_id"]))
+                .collect();
+        }
+    }
+    Vec::new()
+}
+
+#[cfg(test)]
+#[path = "clickup_tests.rs"]
+mod tests;