have different impl's for VNode

src/lib/src/api/client/tree.rs

@@ -11,8 +11,8 @@ use std::time;
 use crate::api::client;
 use crate::constants::{NODES_DIR, OXEN_HIDDEN_DIR, TREE_DIR};
 use crate::core::v_latest::index::merkle_node_db::node_db_path;
-use crate::core::v_latest::structs::PushProgress;
 use crate::core::v_latest::index::CommitMerkleTree;
+use crate::core::v_latest::structs::PushProgress;
 use crate::error::OxenError;
 use crate::model::merkle_tree::node::MerkleTreeNode;
 use crate::model::{Commit, LocalRepository, MerkleHash, RemoteRepository};

src/lib/src/command/migrate/m05_optimize_merkle_tree.rs

@@ -453,10 +453,7 @@ fn migrate_dir(
     let mut vnode_nodes: Vec<VNode> = Vec::new();
     for (i, bhash) in bucket_hashes.iter().enumerate() {
         log::debug!("Bucket [{}] for {:x}", i, bhash);
-        let node = VNode {
-            hash: MerkleHash::new(*bhash),
-            ..Default::default()
-        };
+        let node = VNode::new(repo, MerkleHash::new(*bhash), bucket_hashes.len() as u64)?;
         dir_db.add_child(&node)?;
         vnode_nodes.push(node);
     }
@@ -547,7 +544,7 @@ fn migrate_dir(
 
                     // Recurse if it's a directory
                     let mut dir_db =
-                        MerkleNodeDB::open_read_write(repo, &dir_node, Some(vnode.hash))?;
+                        MerkleNodeDB::open_read_write(repo, &dir_node, Some(vnode.hash()))?;
                     migrate_dir(
                         repo,
                         commits,

src/lib/src/constants.rs

@@ -192,10 +192,6 @@ pub const DEFAULT_PAGE_SIZE: usize = 100;
 /// Pagination page number of 1
 pub const DEFAULT_PAGE_NUM: usize = 1;
 
-/// Redis queue name for post commit actions
-// pub const COMMIT_QUEUE_NAME: &str = "commit_queue";
-// pub const DEFAULT_REDIS_URL: &str = "redis://localhost:6379";
-
 /// Data Types
 pub const TEXT: &str = "text";
 pub const IMAGE: &str = "image";

src/lib/src/core/df/sql.rs

@@ -1,7 +1,7 @@
 use std::path::{Path, PathBuf};
 
-use crate::core::v_latest::workspaces;
 use crate::core::df::tabular;
+use crate::core::v_latest::workspaces;
 use crate::model::LocalRepository;
 use crate::opts::DFOpts;
 use crate::repositories;
@@ -26,9 +26,7 @@ pub fn query_df_from_repo(
     }
 
     let workspace =
-        workspaces::data_frames::get_queryable_data_frame_workspace(
-            repo, path, &commit,
-        )?;
+        workspaces::data_frames::get_queryable_data_frame_workspace(repo, path, &commit)?;
 
     let db_path = repositories::workspaces::data_frames::duckdb_path(&workspace, path);
     let mut conn = df_db::get_connection(db_path)?;

src/lib/src/core/v0_19_0/entries.rs

@@ -1,9 +1,6 @@
-
 use crate::error::OxenError;
 use crate::model::merkle_tree::node::{EMerkleTreeNode, FileNode, MerkleTreeNode};
-use crate::model::{
-    Commit, LocalRepository
-};
+use crate::model::{Commit, LocalRepository};
 use std::path::Path;
 
 use crate::core::v_latest::index::CommitMerkleTree;

src/lib/src/core/v0_19_0/model.rs

@@ -1,2 +1 @@
-
 pub mod merkle_tree;

src/lib/src/core/v0_19_0/model/merkle_tree.rs

@@ -1,2 +1 @@
-
 pub mod node;

src/lib/src/core/v0_19_0/model/merkle_tree/node.rs

@@ -1,2 +1 @@
-
 pub mod vnode;

src/lib/src/core/v_latest/data_frames/schemas.rs

@@ -16,8 +16,8 @@ use std::str;
 use crate::constants;
 use crate::core::db;
 
-use crate::core::v_latest::structs::StagedMerkleTreeNode;
 use crate::core::v_latest::index::CommitMerkleTree;
+use crate::core::v_latest::structs::StagedMerkleTreeNode;
 use crate::error::OxenError;
 use crate::model::merkle_tree::node::EMerkleTreeNode;
 use crate::model::merkle_tree::node::MerkleTreeNode;

src/lib/src/core/v_latest/index/commit_merkle_tree.rs

@@ -569,7 +569,7 @@ impl CommitMerkleTree {
         // And use this to skip to the correct vnode
         let total_children = vnodes
             .iter()
-            .map(|vnode| vnode.vnode().unwrap().num_entries)
+            .map(|vnode| vnode.vnode().unwrap().num_entries())
             .sum::<u64>();
         let vnode_size = repo.vnode_size();
         let num_vnodes = (total_children as f32 / vnode_size as f32).ceil() as u128;

src/lib/src/core/v_latest/index/commit_writer.rs

@@ -18,10 +18,10 @@ use crate::constants::{HEAD_FILE, STAGED_DIR};
 use crate::core::db;
 use crate::core::db::key_val::str_val_db;
 use crate::core::refs::RefWriter;
+use crate::core::v_latest::index::CommitMerkleTree;
 use crate::core::v_latest::index::MerkleNodeDB;
 use crate::core::v_latest::status;
 use crate::core::v_latest::structs::StagedMerkleTreeNode;
-use crate::core::v_latest::index::CommitMerkleTree;
 use crate::error::OxenError;
 use crate::model::merkle_tree::node::EMerkleTreeNode;
 use crate::model::merkle_tree::node::VNode;
@@ -816,11 +816,7 @@ fn r_create_dir_node(
 
     log::debug!("Processing dir {:?} with {} vnodes", path, vnodes.len());
     for vnode in vnodes.iter() {
-        let vnode_obj = VNode {
-            hash: vnode.id,
-            num_entries: vnode.entries.len() as u64,
-            ..Default::default()
-        };
+        let vnode_obj = VNode::new(repo, vnode.id, vnode.entries.len() as u64)?;
         if let Some(dir_db) = maybe_dir_db {
             dir_db.add_child(&vnode_obj)?;
             *total_written += 1;

src/lib/src/core/v_latest/index/merkle_node_db.rs

@@ -196,6 +196,7 @@ pub struct MerkleNodeDB {
     pub node_id: MerkleHash,
     pub parent_id: Option<MerkleHash>,
     read_only: bool,
+    repo: LocalRepository,
     path: PathBuf,
     node_file: Option<File>,
     children_file: Option<File>,
@@ -223,17 +224,23 @@ impl MerkleNodeDB {
     }
 
     pub fn node(&self) -> Result<EMerkleTreeNode, OxenError> {
-        Self::to_node(self.dtype, &self.data())
+        Self::to_node(&self.repo, self.dtype, &self.data())
     }
 
-    pub fn to_node(dtype: MerkleTreeNodeType, data: &[u8]) -> Result<EMerkleTreeNode, OxenError> {
+    pub fn to_node(
+        repo: &LocalRepository,
+        dtype: MerkleTreeNodeType,
+        data: &[u8],
+    ) -> Result<EMerkleTreeNode, OxenError> {
         match dtype {
             MerkleTreeNodeType::Commit => {
                 Ok(EMerkleTreeNode::Commit(CommitNode::deserialize(data)?))
             }
             MerkleTreeNodeType::Dir => Ok(EMerkleTreeNode::Directory(DirNode::deserialize(data)?)),
             MerkleTreeNodeType::File => Ok(EMerkleTreeNode::File(FileNode::deserialize(data)?)),
-            MerkleTreeNodeType::VNode => Ok(EMerkleTreeNode::VNode(VNode::deserialize(data)?)),
+            MerkleTreeNodeType::VNode => {
+                Ok(EMerkleTreeNode::VNode(VNode::deserialize(repo, data)?))
+            }
             MerkleTreeNodeType::FileChunk => Ok(EMerkleTreeNode::FileChunk(
                 FileChunkNode::deserialize(data)?,
             )),
@@ -251,7 +258,7 @@ impl MerkleNodeDB {
 
     pub fn open_read_only(repo: &LocalRepository, hash: &MerkleHash) -> Result<Self, OxenError> {
         let path = node_db_path(repo, hash);
-        Self::open(path, true)
+        Self::open(repo, path, true)
     }
 
     pub fn open_read_write_if_not_exists(
@@ -281,12 +288,16 @@ impl MerkleNodeDB {
             util::fs::create_dir_all(&path)?;
         }
         log::debug!("open_read_write merkle node db at {}", path.display());
-        let mut db = Self::open(path, false)?;
+        let mut db = Self::open(repo, path, false)?;
         db.write_node(node, parent_id)?;
         Ok(db)
     }
 
-    pub fn open(path: impl AsRef<Path>, read_only: bool) -> Result<Self, OxenError> {
+    pub fn open(
+        repo: &LocalRepository,
+        path: impl AsRef<Path>,
+        read_only: bool,
+    ) -> Result<Self, OxenError> {
         let path = path.as_ref();
 
         // mkdir if not exists
@@ -329,6 +340,7 @@ impl MerkleNodeDB {
         let parent_id = lookup.as_ref().map(|l| l.parent_id);
         Ok(Self {
             read_only,
+            repo: repo.clone(),
             path: path.to_path_buf(),
             node_file,
             children_file,
@@ -499,7 +511,7 @@ impl MerkleNodeDB {
 
         // Parse the node parent id
         let data_type = MerkleTreeNodeType::from_u8(lookup.data_type);
-        let parent_id = MerkleTreeNode::deserialize_id(&lookup.data, data_type)?;
+        let parent_id = MerkleTreeNode::deserialize_id(&self.repo, &lookup.data, data_type)?;
 
         let mut file_data = Vec::new();
         children_file.read_to_end(&mut file_data)?;
@@ -521,7 +533,7 @@ impl MerkleNodeDB {
             let node = MerkleTreeNode {
                 parent_id: Some(parent_id),
                 hash: MerkleHash::new(*hash),
-                node: Self::to_node(dtype, &data)?,
+                node: Self::to_node(&self.repo, dtype, &data)?,
                 children: Vec::new(),
             };
             // log::debug!("Loaded node {:?}", node);

src/lib/src/core/v_latest/push.rs

@@ -10,8 +10,8 @@ use crate::model::merkle_tree::node::EMerkleTreeNode;
 use crate::model::{Branch, Commit, CommitEntry, LocalRepository, MerkleHash, RemoteRepository};
 use crate::{api, repositories};
 
-use crate::core::v_latest::structs::push_progress::PushProgress;
 use crate::core::v_latest::index::CommitMerkleTree;
+use crate::core::v_latest::structs::push_progress::PushProgress;
 use crate::model::merkle_tree::node::MerkleTreeNode;
 
 pub async fn push(repo: &LocalRepository) -> Result<Branch, OxenError> {

src/lib/src/core/v_latest/workspaces/data_frames.rs

@@ -5,9 +5,9 @@ use crate::constants::{DIFF_HASH_COL, DIFF_STATUS_COL, EXCLUDE_OXEN_COLS, STAGED
 use crate::core::db;
 use crate::core::db::data_frames::df_db;
 use crate::core::v_latest::add::add_dir_to_staged_db;
+use crate::core::v_latest::index::CommitMerkleTree;
 use crate::core::v_latest::structs::StagedMerkleTreeNode;
 use crate::core::v_latest::workspaces::files::track_modified_data_frame;
-use crate::core::v_latest::index::CommitMerkleTree;
 use rmp_serde::Serializer;
 use serde::Serialize;
 use sql_query_builder::Delete;

src/lib/src/core/v_latest/workspaces/data_frames/columns.rs

@@ -3,9 +3,9 @@ use rocksdb::DB;
 
 use crate::constants::TABLE_NAME;
 use crate::core::db;
-use crate::core::v_latest::data_frames;
 use crate::core::db::data_frames::workspace_df_db::schema_without_oxen_cols;
 use crate::core::db::data_frames::{column_changes_db, columns, df_db};
+use crate::core::v_latest::data_frames;
 use crate::core::v_latest::structs::StagedMerkleTreeNode;
 use crate::core::v_latest::workspaces;
 use crate::error::OxenError;

src/lib/src/core/v_latest/workspaces/files.rs

@@ -9,8 +9,8 @@ use crate::constants::STAGED_DIR;
 use crate::constants::VERSIONS_DIR;
 use crate::core::db;
 use crate::core::v_latest::add::{add_file_node_to_staged_db, process_add_file};
-use crate::core::v_latest::structs::StagedMerkleTreeNode;
 use crate::core::v_latest::index::CommitMerkleTree;
+use crate::core::v_latest::structs::StagedMerkleTreeNode;
 use crate::error::OxenError;
 use crate::model::workspace::Workspace;
 use crate::model::LocalRepository;

src/lib/src/model/merkle_tree/node.rs

@@ -50,7 +50,7 @@ impl EMerkleTreeNode {
         match self {
             EMerkleTreeNode::File(file) => file.hash,
             EMerkleTreeNode::Directory(dir) => dir.hash,
-            EMerkleTreeNode::VNode(vnode) => vnode.hash,
+            EMerkleTreeNode::VNode(vnode) => vnode.hash(),
             EMerkleTreeNode::FileChunk(file_chunk) => file_chunk.hash,
             EMerkleTreeNode::Commit(commit) => commit.hash,
         }

src/lib/src/model/merkle_tree/node/merkle_tree_node.rs

@@ -403,10 +403,14 @@ impl MerkleTreeNode {
         self.node.clone()
     }
 
-    pub fn deserialize_id(data: &[u8], dtype: MerkleTreeNodeType) -> Result<MerkleHash, OxenError> {
+    pub fn deserialize_id(
+        repo: &LocalRepository,
+        data: &[u8],
+        dtype: MerkleTreeNodeType,
+    ) -> Result<MerkleHash, OxenError> {
         match dtype {
             MerkleTreeNodeType::Commit => CommitNode::deserialize(data).map(|commit| commit.hash),
-            MerkleTreeNodeType::VNode => VNode::deserialize(data).map(|vnode| vnode.hash),
+            MerkleTreeNodeType::VNode => VNode::deserialize(repo, data).map(|vnode| vnode.hash()),
             MerkleTreeNodeType::Dir => DirNode::deserialize(data).map(|dir| dir.hash),
             MerkleTreeNodeType::File => FileNode::deserialize(data).map(|file| file.hash),
             MerkleTreeNodeType::FileChunk => {