Improve support for sparse cache.

With the addition of a notes backed cache we will have the
possibility of avoiding a "cold start" by transferring just
the notes to another repo. To keep the amount of extra data
small the notes cache is kept sparse: Not all commits have
and entry and also in shards by sequence number, as old
entries are unlikely to be relevant.
The old traversal logic did not perform very well with sparse
cache. Especially "find_known" proved to be a bottleneck.
So the traversal is revised now to work better in the sparse
cache case.

Change: cache-sparse-shards

Author: christian-schilling

josh-core/src/cache.rs

@@ -9,9 +9,20 @@ use std::sync::{LazyLock, RwLock};
 pub(crate) const CACHE_VERSION: u64 = 24;
 
 pub trait CacheBackend: Send + Sync {
-    fn read(&self, filter: filter::Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>>;
-
-    fn write(&self, filter: filter::Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()>;
+    fn read(
+        &self,
+        filter: filter::Filter,
+        from: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>>;
+
+    fn write(
+        &self,
+        filter: filter::Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<()>;
 }
 
 pub trait FilterHook {
@@ -323,6 +334,11 @@ impl Transaction {
     }
 
     pub fn insert(&self, filter: filter::Filter, from: git2::Oid, to: git2::Oid, store: bool) {
+        let sequence_number = if filter != filter::sequence_number() {
+            compute_sequence_number(self, from).expect("compute_sequence_number failed")
+        } else {
+            0
+        };
         let mut t2 = self.t2.borrow_mut();
         t2.commit_map
             .entry(filter.id())
@@ -334,14 +350,13 @@ impl Transaction {
         // the history length by a very large factor.
         if store || from.as_bytes()[0] == 0 {
             t2.cache
-                .write_all(filter, from, to)
+                .write_all(filter, from, to, sequence_number)
                 .expect("Failed to write cache");
         }
     }
 
     pub fn get_missing(&self) -> Vec<(filter::Filter, git2::Oid)> {
         let mut missing = self.t2.borrow().missing.clone();
-        missing.sort_by_key(|(f, i)| (filter::nesting(*f), *f, *i));
         missing.dedup();
         missing.retain(|(f, i)| !self.known(*f, *i));
         self.t2.borrow_mut().missing = missing.clone();
@@ -358,7 +373,9 @@ impl Transaction {
         } else {
             let mut t2 = self.t2.borrow_mut();
             t2.misses += 1;
-            t2.missing.push((filter, from));
+            if !t2.missing.contains(&(filter, from)) {
+                t2.missing.insert(0, (filter, from));
+            }
             None
         }
     }
@@ -367,6 +384,11 @@ impl Transaction {
         if filter == filter::nop() {
             return Some(from);
         }
+        let sequence_number = if filter != filter::sequence_number() {
+            compute_sequence_number(self, from).expect("compute_sequence_number failed")
+        } else {
+            0
+        };
         let t2 = self.t2.borrow_mut();
         if let Some(m) = t2.commit_map.get(&filter.id()) {
             if let Some(oid) = m.get(&from).cloned() {
@@ -376,7 +398,7 @@ impl Transaction {
 
         let oid = t2
             .cache
-            .read_propagate(filter, from)
+            .read_propagate(filter, from, sequence_number)
             .expect("Failed to read from cache backend");
 
         let oid = if let Some(oid) = oid { Some(oid) } else { None };
@@ -385,6 +407,9 @@ impl Transaction {
             if oid == git2::Oid::zero() {
                 return Some(oid);
             }
+            if filter == filter::sequence_number() {
+                return Some(oid);
+            }
 
             if self.repo.odb().unwrap().exists(oid) {
                 // Only report an object as cached if it exists in the object database.
@@ -396,3 +421,70 @@ impl Transaction {
         None
     }
 }
+
+/// Encode a `u128` into a 20-byte git OID (SHA-1 sized).
+/// The high 4 bytes of the OID are zero; the low 16 bytes
+/// contain the big-endian integer.
+pub fn oid_from_u128(n: u128) -> git2::Oid {
+    let mut bytes = [0u8; 20];
+    // place the 16 integer bytes at the end (big-endian)
+    bytes[20 - 16..].copy_from_slice(&n.to_be_bytes());
+    // Safe: length is exactly 20
+    git2::Oid::from_bytes(&bytes).expect("20-byte OID construction cannot fail")
+}
+
+/// Decode a `u128` previously encoded by `oid_from_u128`.
+pub fn u128_from_oid(oid: git2::Oid) -> u128 {
+    let b = oid.as_bytes();
+    let mut n = [0u8; 16];
+    n.copy_from_slice(&b[20 - 16..]); // take the last 16 bytes
+    u128::from_be_bytes(n)
+}
+
+pub fn compute_sequence_number(
+    transaction: &cache::Transaction,
+    input: git2::Oid,
+) -> JoshResult<u128> {
+    if let Some(count) = transaction.get(filter::sequence_number(), input) {
+        return Ok(u128_from_oid(count));
+    }
+
+    let commit = transaction.repo().find_commit(input)?;
+    if let Some(p) = commit.parent_ids().next() {
+        if let Some(count) = transaction.get(filter::sequence_number(), p) {
+            let pc = u128_from_oid(count);
+            transaction.insert(
+                filter::sequence_number(),
+                input,
+                oid_from_u128(pc + 1),
+                true,
+            );
+            return Ok(pc + 1);
+        }
+    }
+
+    let mut walk = transaction.repo().revwalk()?;
+    walk.set_sorting(git2::Sort::REVERSE | git2::Sort::TOPOLOGICAL)?;
+    walk.push(input)?;
+
+    for c in walk {
+        let commit = transaction.repo().find_commit(c?)?;
+        let pc = if let Some(p) = commit.parent_ids().next() {
+            compute_sequence_number(transaction, p)?
+        } else {
+            0
+        };
+
+        transaction.insert(
+            filter::sequence_number(),
+            commit.id(),
+            oid_from_u128(pc + 1),
+            true,
+        );
+    }
+    if let Some(count) = transaction.get(filter::sequence_number(), input) {
+        Ok(u128_from_oid(count))
+    } else {
+        Err(josh_error("missing sequence_number"))
+    }
+}

josh-core/src/cache_notes.rs

@@ -1,5 +1,6 @@
 use crate::JoshResult;
 use crate::cache::{CACHE_VERSION, CacheBackend};
+use crate::filter;
 use crate::filter::Filter;
 
 pub struct NotesCacheBackend {
@@ -15,24 +16,53 @@ impl NotesCacheBackend {
     }
 }
 
-fn is_note_eligible(oid: git2::Oid) -> bool {
-    oid.as_bytes()[0] == 0
+// The notes cache is meant to be sparse. That is, not all entries are actually persisted.
+// This makes it smaller and faster to download.
+// It is expected that on any node (server, proxy, local repo) a full "dense" local cache
+// is used in addition to the sparse note cache.
+// The note cache is mostly only used for initial "cold starts" or longer "catch up".
+// For incremental filtering it's fine re-filter commits and rely on the local "dense" cache.
+// We store entries for 1% of all commits, and additionally all merges and orphans.
+fn is_note_eligible(repo: &git2::Repository, oid: git2::Oid, sequence_number: u128) -> bool {
+    let parent_count = if let Ok(c) = repo.find_commit(oid) {
+        c.parent_ids().count()
+    } else {
+        return false;
+    };
+
+    sequence_number % 100 == 0 || parent_count != 1
 }
 
-fn note_path(key: git2::Oid) -> String {
-    format!("refs/josh/{}/{}", CACHE_VERSION, key)
+// To additionally limit the size of the note trees the cache is also sharded by sequence
+// number in groups of 10000. Note that this does not limit the number of entried per bucket
+// as branches mean many commits share the same sequence number.
+fn note_path(key: git2::Oid, sequence_number: u128) -> String {
+    format!(
+        "refs/josh/{}/{}/{}",
+        CACHE_VERSION,
+        sequence_number / 10000,
+        key,
+    )
 }
 
 impl CacheBackend for NotesCacheBackend {
-    fn read(&self, filter: Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>> {
+    fn read(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>> {
+        if filter == filter::sequence_number() {
+            return Ok(None);
+        }
         let repo = self.repo.lock()?;
-        let key = crate::filter::as_tree(&repo, filter)?;
-
-        if !is_note_eligible(from) {
+        if !is_note_eligible(&repo, from, sequence_number) {
             return Ok(None);
         }
 
-        if let Ok(note) = repo.find_note(Some(&note_path(key)), from) {
+        let key = crate::filter::as_tree(&*repo, filter)?;
+
+        if let Ok(note) = repo.find_note(Some(&note_path(key, sequence_number)), from) {
             let message = note.message().unwrap_or("");
             let result = git2::Oid::from_str(message)?;
 
@@ -42,20 +72,29 @@ impl CacheBackend for NotesCacheBackend {
         }
     }
 
-    fn write(&self, filter: Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()> {
-        let repo = self.repo.lock()?;
-        let key = crate::filter::as_tree(&repo, filter)?;
+    fn write(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<()> {
+        if filter == filter::sequence_number() {
+            return Ok(());
+        }
 
-        if !is_note_eligible(from) {
+        let repo = self.repo.lock()?;
+        if !is_note_eligible(&*repo, from, sequence_number) {
             return Ok(());
         }
 
+        let key = crate::filter::as_tree(&*repo, filter)?;
         let signature = crate::cache::josh_commit_signature()?;
 
         repo.note(
             &signature,
             &signature,
-            Some(&note_path(key)),
+            Some(&note_path(key, sequence_number)),
             from,
             &to.to_string(),
             true,

josh-core/src/cache_sled.rs

@@ -80,7 +80,12 @@ fn insert_sled_tree(filter: Filter) -> sled::Tree {
 }
 
 impl CacheBackend for SledCacheBackend {
-    fn read(&self, filter: Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>> {
+    fn read(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        _sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>> {
         let mut trees = self.trees.lock()?;
         let tree = trees
             .entry(filter.id())
@@ -94,7 +99,13 @@ impl CacheBackend for SledCacheBackend {
         }
     }
 
-    fn write(&self, filter: Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()> {
+    fn write(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        _sequence_number: u128,
+    ) -> JoshResult<()> {
         let mut trees = self.trees.lock()?;
         let tree = trees
             .entry(filter.id())

josh-core/src/cache_stack.rs

@@ -33,9 +33,10 @@ impl CacheStack {
         filter: filter::Filter,
         from: git2::Oid,
         to: git2::Oid,
+        sequence_number: u128,
     ) -> JoshResult<()> {
         for backend in &self.backends {
-            backend.write(filter, from, to)?;
+            backend.write(filter, from, to, sequence_number)?;
         }
 
         Ok(())
@@ -51,16 +52,19 @@ impl CacheStack {
         &self,
         filter: filter::Filter,
         from: git2::Oid,
+        sequence_number: u128,
     ) -> JoshResult<Option<git2::Oid>> {
         let values = self
             .backends
             .iter()
             .enumerate()
-            .find_map(|(index, backend)| match backend.read(filter, from) {
-                Ok(None) => None,
-                Ok(Some(oid)) => Some(Ok((index, oid))),
-                Err(e) => Some(Err(e)),
-            });
+            .find_map(
+                |(index, backend)| match backend.read(filter, from, sequence_number) {
+                    Ok(None) => None,
+                    Ok(Some(oid)) => Some(Ok((index, oid))),
+                    Err(e) => Some(Err(e)),
+                },
+            );
 
         let (index, oid) = match values {
             // None of the backends had the value
@@ -74,7 +78,7 @@ impl CacheStack {
         self.backends
             .iter()
             .take(index)
-            .try_for_each(|backend| backend.write(filter, from, oid))?;
+            .try_for_each(|backend| backend.write(filter, from, oid, sequence_number))?;
 
         Ok(Some(oid))
     }