Speed up random access by file name by changing how we index into vnodes

src/cli/src/cmd/delete_remote.rs

@@ -34,6 +34,12 @@ impl RunCmd for DeleteRemoteCmd {
                 .help("The host you want to create the remote repository on. For example: 'hub.oxen.ai'")
                 .action(clap::ArgAction::Set),
         )
+        .arg(
+            Arg::new("scheme")
+                .long("scheme")
+                .help("The scheme for the url of the remote repository. For example: 'https' or 'http'")
+                .action(clap::ArgAction::Set),
+        )
         .arg(
             Arg::new("yes")
                 .long("yes")
@@ -55,10 +61,14 @@ impl RunCmd for DeleteRemoteCmd {
             .get_one::<String>("host")
             .map(String::from)
             .unwrap_or(DEFAULT_HOST.to_string());
+        // Default scheme
+        let scheme = args
+            .get_one::<String>("scheme")
+            .map(String::from)
+            .unwrap_or("https".to_string());
 
-        let Some(remote_repo) =
-            api::client::repositories::get_by_name_and_host(namespace_name, host).await?
-        else {
+        let url = format!("{}://{host}/{namespace_name}", scheme);
+        let Some(remote_repo) = api::client::repositories::get_by_url(&url).await? else {
             return Err(OxenError::basic_str(format!(
                 "Remote repository not found: {namespace_name}"
             )));

src/cli/src/cmd/node.rs

@@ -3,6 +3,7 @@ use clap::{Arg, Command};
 use liboxen::core::v0_19_0::index::CommitMerkleTree;
 use liboxen::error::OxenError;
 use liboxen::model::{LocalRepository, MerkleHash};
+use liboxen::repositories;
 
 use std::str::FromStr;
 
@@ -20,7 +21,6 @@ impl RunCmd for NodeCmd {
         // Setups the CLI args for the command
         Command::new(NAME)
             .about("Inspect an oxen merkle tree node")
-            .arg(Arg::new("node").required(true).action(clap::ArgAction::Set))
             // add --verbose flag
             .arg(
                 Arg::new("verbose")
@@ -29,13 +29,36 @@ impl RunCmd for NodeCmd {
                     .help("Verbose output")
                     .action(clap::ArgAction::SetTrue),
             )
+            // add --node flag
+            .arg(
+                Arg::new("node")
+                    .long("node")
+                    .short('n')
+                    .help("Node hash to inspect"),
+            )
+            // add --file flag
+            .arg(
+                Arg::new("file")
+                    .long("file")
+                    .short('f')
+                    .help("File path to inspect"),
+            )
     }
 
     async fn run(&self, args: &clap::ArgMatches) -> Result<(), OxenError> {
-        // Parse Args
-        let node_hash = args.get_one::<String>("node").expect("Must supply node");
-
+        // Find the repository
         let repository = LocalRepository::from_current_dir()?;
+
+        // if the --file flag is set, we need to get the node for the file
+        if let Some(file) = args.get_one::<String>("file") {
+            let commit = repositories::commits::head_commit(&repository)?;
+            let node = repositories::entries::get_file(&repository, &commit, file)?;
+            println!("{:?}", node);
+            return Ok(());
+        }
+
+        // otherwise, get the node based on the node hash
+        let node_hash = args.get_one::<String>("node").expect("Must supply node");
         let node_hash = MerkleHash::from_str(node_hash)?;
         let node = CommitMerkleTree::read_node(&repository, &node_hash, false)?;
 

src/lib/src/api/client/tree.rs

@@ -5,12 +5,14 @@ use flate2::Compression;
 use futures_util::TryStreamExt;
 use std::collections::HashSet;
 use std::path::PathBuf;
+use std::sync::Arc;
 use std::time;
 
 use crate::api::client;
 use crate::constants::{NODES_DIR, OXEN_HIDDEN_DIR, TREE_DIR};
 use crate::core::v0_19_0::index::merkle_node_db::node_db_path;
 use crate::core::v0_19_0::index::CommitMerkleTree;
+use crate::core::v0_19_0::structs::PushProgress;
 use crate::error::OxenError;
 use crate::model::merkle_tree::node::MerkleTreeNode;
 use crate::model::{Commit, LocalRepository, MerkleHash, RemoteRepository};
@@ -52,11 +54,13 @@ pub async fn create_nodes(
     local_repo: &LocalRepository,
     remote_repo: &RemoteRepository,
     nodes: HashSet<MerkleTreeNode>,
+    progress: &Arc<PushProgress>,
 ) -> Result<(), OxenError> {
     // Compress the node
     let enc = GzEncoder::new(Vec::new(), Compression::default());
     let mut tar = tar::Builder::new(enc);
-    for node in nodes {
+    let mut children_count = 0;
+    for (i, node) in nodes.iter().enumerate() {
         let node_dir = node_db_path(local_repo, &node.hash);
         let tree_dir = local_repo
             .path
@@ -69,7 +73,15 @@ pub async fn create_nodes(
             sub_dir,
             node_dir
         );
+        progress.set_message(format!(
+            "Packing {}/{} nodes with {} children",
+            i + 1,
+            nodes.len(),
+            children_count
+        ));
+
         tar.append_dir_all(sub_dir, node_dir)?;
+        children_count += node.children.len();
     }
 
     tar.finish()?;

src/lib/src/core/refs/ref_reader.rs

@@ -161,7 +161,7 @@ impl RefReader {
     }
 
     pub fn get_branch_by_name(&self, name: &str) -> Result<Option<Branch>, OxenError> {
-        log::debug!("get_branch_by_name {name}");
+        // log::debug!("get_branch_by_name {name}");
         match self.get_commit_id_for_branch(name) {
             Ok(Some(commit_id)) => Ok(Some(Branch {
                 name: name.to_string(),

src/lib/src/core/v0_19_0/branches.rs

@@ -421,8 +421,10 @@ pub fn restore_file(
     let version_path = util::fs::version_path_from_hash(repo, file_node.hash.to_string());
     if !version_path.exists() {
         return Err(OxenError::basic_str(format!(
-            "Source file not found in versions directory: {:?}",
-            version_path
+            "Restoring file, source file {} not found in versions directory: {:?} when restoring to {:?}",
+            file_node.name,
+            version_path,
+            dst_path
         )));
     }
 

src/lib/src/core/v0_19_0/commits.rs

@@ -156,10 +156,10 @@ pub fn get_by_id(
 ) -> Result<Option<Commit>, OxenError> {
     let commit_id_str = commit_id_str.as_ref();
     let Ok(commit_id) = MerkleHash::from_str(commit_id_str) else {
-        log::debug!(
-            "get_by_id could not create commit_id from [{}]",
-            commit_id_str
-        );
+        // log::debug!(
+        //     "get_by_id could not create commit_id from [{}]",
+        //     commit_id_str
+        // );
         return Ok(None);
     };
     get_by_hash(repo, &commit_id)

src/lib/src/core/v0_19_0/entries.rs

@@ -49,19 +49,15 @@ pub fn get_file_merkle_tree_node(
     commit: &Commit,
     path: impl AsRef<Path>,
 ) -> Result<Option<MerkleTreeNode>, OxenError> {
-    let parent = path.as_ref().parent().unwrap_or(Path::new(""));
-    let parent_node = CommitMerkleTree::dir_with_children(repo, commit, parent)?;
-    let Some(parent_node) = parent_node else {
-        log::debug!("path has no parent: {:?}", path.as_ref());
+    let path = path.as_ref();
+    let dir_hashes = CommitMerkleTree::dir_hashes(repo, commit)?;
+    if dir_hashes.contains_key(path) {
+        // This is a directory, not a file
+        log::debug!("get_file_merkle_tree_node path is a directory: {:?}", path);
         return Ok(None);
-    };
-
-    let Some(file_name) = path.as_ref().file_name() else {
-        log::debug!("path has no file name: {:?}", path.as_ref());
-        return Ok(None);
-    };
+    }
 
-    let file_node = parent_node.get_by_path(file_name)?;
+    let file_node = CommitMerkleTree::read_file(repo, &dir_hashes, path)?;
     Ok(file_node)
 }
 
@@ -325,20 +321,20 @@ fn p_dir_entries(
 ) -> Result<(), OxenError> {
     let search_directory = search_directory.as_ref();
     let current_directory = current_directory.as_ref();
-    log::debug!(
-        "p_dir_entries current_directory {:?} search_directory {:?} node {}",
-        current_directory,
-        search_directory,
-        node
-    );
+    // log::debug!(
+    //     "p_dir_entries current_directory {:?} search_directory {:?} node {}",
+    //     current_directory,
+    //     search_directory,
+    //     node
+    // );
     for child in &node.children {
         match &child.node {
             EMerkleTreeNode::VNode(_) => {
-                log::debug!(
-                    "p_dir_entries got vnode {:?} search_directory {:?}",
-                    current_directory,
-                    search_directory
-                );
+                // log::debug!(
+                //     "p_dir_entries got vnode {:?} search_directory {:?}",
+                //     current_directory,
+                //     search_directory
+                // );
                 p_dir_entries(
                     repo,
                     child,
@@ -350,25 +346,25 @@ fn p_dir_entries(
                 )?;
             }
             EMerkleTreeNode::Directory(child_dir) => {
-                log::debug!(
-                    "p_dir_entries current_directory {:?} search_directory {:?} child_dir {:?}",
-                    current_directory,
-                    search_directory,
-                    child_dir.name
-                );
+                // log::debug!(
+                //     "p_dir_entries current_directory {:?} search_directory {:?} child_dir {:?}",
+                //     current_directory,
+                //     search_directory,
+                //     child_dir.name
+                // );
                 if current_directory == search_directory && !child_dir.name.is_empty() {
-                    log::debug!(
-                        "p_dir_entries adding dir entry current_directory {:?}",
-                        current_directory
-                    );
+                    // log::debug!(
+                    //     "p_dir_entries adding dir entry current_directory {:?}",
+                    //     current_directory
+                    // );
                     let metadata = dir_node_to_metadata_entry(
                         repo,
                         child,
                         parsed_resource,
                         found_commits,
                         true,
                     )?;
-                    log::debug!("p_dir_entries added dir entry {:?}", metadata);
+                    // log::debug!("p_dir_entries added dir entry {:?}", metadata);
                     entries.push(metadata.unwrap());
                 }
                 let current_directory = current_directory.join(&child_dir.name);
@@ -382,27 +378,27 @@ fn p_dir_entries(
                     entries,
                 )?;
             }
-            EMerkleTreeNode::File(child_file) => {
-                log::debug!(
-                    "p_dir_entries current_directory {:?} search_directory {:?} child_file {:?}",
-                    current_directory,
-                    search_directory,
-                    child_file.name
-                );
+            EMerkleTreeNode::File(_child_file) => {
+                // log::debug!(
+                //     "p_dir_entries current_directory {:?} search_directory {:?} child_file {:?}",
+                //     current_directory,
+                //     search_directory,
+                //     child_file.name
+                // );
 
                 if current_directory == search_directory {
-                    log::debug!(
-                        "p_dir_entries adding file entry current_directory {:?} file_name {:?}",
-                        current_directory,
-                        child_file.name
-                    );
+                    // log::debug!(
+                    //     "p_dir_entries adding file entry current_directory {:?} file_name {:?}",
+                    //     current_directory,
+                    //     child_file.name
+                    // );
                     let metadata =
                         file_node_to_metadata_entry(repo, child, parsed_resource, found_commits)?;
-                    log::debug!(
-                        "p_dir_entries added file entry {:?} file_name {:?}",
-                        metadata,
-                        child_file.name
-                    );
+                    // log::debug!(
+                    //     "p_dir_entries added file entry {:?} file_name {:?}",
+                    //     metadata,
+                    //     child_file.name
+                    // );
                     entries.push(metadata.unwrap());
                 }
             }

src/lib/src/core/v0_19_0/fetch.rs

@@ -367,6 +367,7 @@ pub async fn maybe_fetch_missing_entries(
 
     // TODO: what should we print here? If there is nothing to pull, we
     // shouldn't show the PullProgress
+    log::debug!("Fetching missing entries for commit {}", commit);
 
     // Keep track of how many bytes we have downloaded
     let pull_progress = Arc::new(PullProgress::new());
@@ -392,6 +393,12 @@ async fn r_download_entries(
     directory: &Path,
     pull_progress: &Arc<PullProgress>,
 ) -> Result<(), OxenError> {
+    log::debug!(
+        "fetch r_download_entries ({}) {:?} {:?}",
+        node.children.len(),
+        node.hash,
+        node.node
+    );
     for child in &node.children {
         let mut new_directory = directory.to_path_buf();
         if let EMerkleTreeNode::Directory(dir_node) = &child.node {