Improve support for sparse cache.

josh-core/src/cache.rs

@@ -9,9 +9,20 @@ use std::sync::{LazyLock, RwLock};
 pub(crate) const CACHE_VERSION: u64 = 24;
 
 pub trait CacheBackend: Send + Sync {
-    fn read(&self, filter: filter::Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>>;
-
-    fn write(&self, filter: filter::Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()>;
+    fn read(
+        &self,
+        filter: filter::Filter,
+        from: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>>;
+
+    fn write(
+        &self,
+        filter: filter::Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<()>;
 }
 
 pub trait FilterHook {
@@ -323,6 +334,11 @@ impl Transaction {
     }
 
     pub fn insert(&self, filter: filter::Filter, from: git2::Oid, to: git2::Oid, store: bool) {
+        let sequence_number = if filter != filter::sequence_number() {
+            compute_sequence_number(self, from).expect("compute_sequence_number failed")
+        } else {
+            0
+        };
         let mut t2 = self.t2.borrow_mut();
         t2.commit_map
             .entry(filter.id())
@@ -334,14 +350,13 @@ impl Transaction {
         // the history length by a very large factor.
         if store || from.as_bytes()[0] == 0 {
             t2.cache
-                .write_all(filter, from, to)
+                .write_all(filter, from, to, sequence_number)
                 .expect("Failed to write cache");
         }
     }
 
     pub fn get_missing(&self) -> Vec<(filter::Filter, git2::Oid)> {
         let mut missing = self.t2.borrow().missing.clone();
-        missing.sort_by_key(|(f, i)| (filter::nesting(*f), *f, *i));
         missing.dedup();
         missing.retain(|(f, i)| !self.known(*f, *i));
         self.t2.borrow_mut().missing = missing.clone();
@@ -358,7 +373,9 @@ impl Transaction {
         } else {
             let mut t2 = self.t2.borrow_mut();
             t2.misses += 1;
-            t2.missing.push((filter, from));
+            if !t2.missing.contains(&(filter, from)) {
+                t2.missing.insert(0, (filter, from));
+            }
             None
         }
     }
@@ -367,6 +384,11 @@ impl Transaction {
         if filter == filter::nop() {
             return Some(from);
         }
+        let sequence_number = if filter != filter::sequence_number() {
+            compute_sequence_number(self, from).expect("compute_sequence_number failed")
+        } else {
+            0
+        };
         let t2 = self.t2.borrow_mut();
         if let Some(m) = t2.commit_map.get(&filter.id()) {
             if let Some(oid) = m.get(&from).cloned() {
@@ -376,7 +398,7 @@ impl Transaction {
 
         let oid = t2
             .cache
-            .read_propagate(filter, from)
+            .read_propagate(filter, from, sequence_number)
             .expect("Failed to read from cache backend");
 
         let oid = if let Some(oid) = oid { Some(oid) } else { None };
@@ -385,6 +407,9 @@ impl Transaction {
             if oid == git2::Oid::zero() {
                 return Some(oid);
             }
+            if filter == filter::sequence_number() {
+                return Some(oid);
+            }
 
             if self.repo.odb().unwrap().exists(oid) {
                 // Only report an object as cached if it exists in the object database.
@@ -396,3 +421,70 @@ impl Transaction {
         None
     }
 }
+
+/// Encode a `u128` into a 20-byte git OID (SHA-1 sized).
+/// The high 4 bytes of the OID are zero; the low 16 bytes
+/// contain the big-endian integer.
+pub fn oid_from_u128(n: u128) -> git2::Oid {
+    let mut bytes = [0u8; 20];
+    // place the 16 integer bytes at the end (big-endian)
+    bytes[20 - 16..].copy_from_slice(&n.to_be_bytes());
+    // Safe: length is exactly 20
+    git2::Oid::from_bytes(&bytes).expect("20-byte OID construction cannot fail")
+}
+
+/// Decode a `u128` previously encoded by `oid_from_u128`.
+pub fn u128_from_oid(oid: git2::Oid) -> u128 {
+    let b = oid.as_bytes();
+    let mut n = [0u8; 16];
+    n.copy_from_slice(&b[20 - 16..]); // take the last 16 bytes
+    u128::from_be_bytes(n)
+}
+
+pub fn compute_sequence_number(
+    transaction: &cache::Transaction,
+    input: git2::Oid,
+) -> JoshResult<u128> {
+    if let Some(count) = transaction.get(filter::sequence_number(), input) {
+        return Ok(u128_from_oid(count));
+    }
+
+    let commit = transaction.repo().find_commit(input)?;
+    if let Some(p) = commit.parent_ids().next() {
+        if let Some(count) = transaction.get(filter::sequence_number(), p) {
+            let pc = u128_from_oid(count);
+            transaction.insert(
+                filter::sequence_number(),
+                input,
+                oid_from_u128(pc + 1),
+                true,
+            );
+            return Ok(pc + 1);
+        }
+    }
+
+    let mut walk = transaction.repo().revwalk()?;
+    walk.set_sorting(git2::Sort::REVERSE | git2::Sort::TOPOLOGICAL)?;
+    walk.push(input)?;
+
+    for c in walk {
+        let commit = transaction.repo().find_commit(c?)?;
+        let pc = if let Some(p) = commit.parent_ids().next() {
+            compute_sequence_number(transaction, p)?
+        } else {
+            0
+        };
+
+        transaction.insert(
+            filter::sequence_number(),
+            commit.id(),
+            oid_from_u128(pc + 1),
+            true,
+        );
+    }
+    if let Some(count) = transaction.get(filter::sequence_number(), input) {
+        Ok(u128_from_oid(count))
+    } else {
+        Err(josh_error("missing sequence_number"))
+    }
+}

josh-core/src/cache_notes.rs

@@ -1,5 +1,6 @@
 use crate::JoshResult;
 use crate::cache::{CACHE_VERSION, CacheBackend};
+use crate::filter;
 use crate::filter::Filter;
 
 pub struct NotesCacheBackend {
@@ -15,24 +16,53 @@ impl NotesCacheBackend {
     }
 }
 
-fn is_note_eligible(oid: git2::Oid) -> bool {
-    oid.as_bytes()[0] == 0
+// The notes cache is meant to be sparse. That is, not all entries are actually persisted.
+// This makes it smaller and faster to download.
+// It is expected that on any node (server, proxy, local repo) a full "dense" local cache
+// is used in addition to the sparse note cache.
+// The note cache is mostly only used for initial "cold starts" or longer "catch up".
+// For incremental filtering it's fine re-filter commits and rely on the local "dense" cache.
+// We store entries for 1% of all commits, and additionally all merges and orphans.
+fn is_note_eligible(repo: &git2::Repository, oid: git2::Oid, sequence_number: u128) -> bool {
+    let parent_count = if let Ok(c) = repo.find_commit(oid) {
+        c.parent_ids().count()
+    } else {
+        return false;
+    };
+
+    sequence_number % 100 == 0 || parent_count != 1
 }
 
-fn note_path(key: git2::Oid) -> String {
-    format!("refs/josh/{}/{}", CACHE_VERSION, key)
+// To additionally limit the size of the note trees the cache is also sharded by sequence
+// number in groups of 10000. Note that this does not limit the number of entried per bucket
+// as branches mean many commits share the same sequence number.
+fn note_path(key: git2::Oid, sequence_number: u128) -> String {
+    format!(
+        "refs/josh/{}/{}/{}",
+        CACHE_VERSION,
+        sequence_number / 10000,
+        key,
+    )
 }
 
 impl CacheBackend for NotesCacheBackend {
-    fn read(&self, filter: Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>> {
+    fn read(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>> {
+        if filter == filter::sequence_number() {
+            return Ok(None);
+        }
         let repo = self.repo.lock()?;
-        let key = crate::filter::as_tree(&repo, filter)?;
-
-        if !is_note_eligible(from) {
+        if !is_note_eligible(&repo, from, sequence_number) {
             return Ok(None);
         }
 
-        if let Ok(note) = repo.find_note(Some(&note_path(key)), from) {
+        let key = crate::filter::as_tree(&*repo, filter)?;
+
+        if let Ok(note) = repo.find_note(Some(&note_path(key, sequence_number)), from) {
             let message = note.message().unwrap_or("");
             let result = git2::Oid::from_str(message)?;
 
@@ -42,20 +72,29 @@ impl CacheBackend for NotesCacheBackend {
         }
     }
 
-    fn write(&self, filter: Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()> {
-        let repo = self.repo.lock()?;
-        let key = crate::filter::as_tree(&repo, filter)?;
+    fn write(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<()> {
+        if filter == filter::sequence_number() {
+            return Ok(());
+        }
 
-        if !is_note_eligible(from) {
+        let repo = self.repo.lock()?;
+        if !is_note_eligible(&*repo, from, sequence_number) {
             return Ok(());
         }
 
+        let key = crate::filter::as_tree(&*repo, filter)?;
         let signature = crate::cache::josh_commit_signature()?;
 
         repo.note(
             &signature,
             &signature,
-            Some(&note_path(key)),
+            Some(&note_path(key, sequence_number)),
             from,
             &to.to_string(),
             true,

josh-core/src/cache_sled.rs

@@ -80,7 +80,12 @@ fn insert_sled_tree(filter: Filter) -> sled::Tree {
 }
 
 impl CacheBackend for SledCacheBackend {
-    fn read(&self, filter: Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>> {
+    fn read(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        _sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>> {
         let mut trees = self.trees.lock()?;
         let tree = trees
             .entry(filter.id())
@@ -94,7 +99,13 @@ impl CacheBackend for SledCacheBackend {
         }
     }
 
-    fn write(&self, filter: Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()> {
+    fn write(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        _sequence_number: u128,
+    ) -> JoshResult<()> {
         let mut trees = self.trees.lock()?;
         let tree = trees
             .entry(filter.id())

josh-core/src/cache_stack.rs

@@ -33,9 +33,10 @@ impl CacheStack {
         filter: filter::Filter,
         from: git2::Oid,
         to: git2::Oid,
+        sequence_number: u128,
     ) -> JoshResult<()> {
         for backend in &self.backends {
-            backend.write(filter, from, to)?;
+            backend.write(filter, from, to, sequence_number)?;
         }
 
         Ok(())
@@ -51,16 +52,19 @@ impl CacheStack {
         &self,
         filter: filter::Filter,
         from: git2::Oid,
+        sequence_number: u128,
     ) -> JoshResult<Option<git2::Oid>> {
         let values = self
             .backends
             .iter()
             .enumerate()
-            .find_map(|(index, backend)| match backend.read(filter, from) {
-                Ok(None) => None,
-                Ok(Some(oid)) => Some(Ok((index, oid))),
-                Err(e) => Some(Err(e)),
-            });
+            .find_map(
+                |(index, backend)| match backend.read(filter, from, sequence_number) {
+                    Ok(None) => None,
+                    Ok(Some(oid)) => Some(Ok((index, oid))),
+                    Err(e) => Some(Err(e)),
+                },
+            );
 
         let (index, oid) = match values {
             // None of the backends had the value
@@ -74,7 +78,7 @@ impl CacheStack {
         self.backends
             .iter()
             .take(index)
-            .try_for_each(|backend| backend.write(filter, from, oid))?;
+            .try_for_each(|backend| backend.write(filter, from, oid, sequence_number))?;
 
         Ok(Some(oid))
     }