From ca41f3a9be46db900aa70e6bbb78da7553e0b808 Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Sat, 24 Jan 2026 00:04:36 -0800 Subject: [PATCH 01/49] initial commit for backend refactor --- Cargo.lock | 58 +- rust/Cargo.toml | 6 +- rust/src/diff/differ.rs | 196 ++-- rust/src/diff/resource_differ.rs | 29 +- rust/src/diff/scene_differ.rs | 54 +- rust/src/diff/text_differ.rs | 2 +- rust/src/fs.rs | 4 +- rust/src/fs/file_system_driver.rs | 1355 +++++++++++----------- rust/src/fs/file_system_driver/tests.rs | 836 ++++++------- rust/src/fs/file_utils.rs | 53 +- rust/src/helpers/utils.rs | 80 +- rust/src/interop/godot_project.rs | 23 +- rust/src/project.rs | 15 +- rust/src/project/branch_db.rs | 168 +++ rust/src/project/branch_db/branch.rs | 99 ++ rust/src/project/branch_db/commit.rs | 254 ++++ rust/src/project/branch_db/file.rs | 284 +++++ rust/src/project/branch_db/util.rs | 79 ++ rust/src/project/connection.rs | 162 +++ rust/src/project/document_watcher.rs | 231 ++++ rust/src/project/driver.rs | 483 ++++++++ rust/src/project/fs_watcher.rs | 230 ++++ rust/src/project/new_project.rs | 294 +++++ rust/src/project/peer_watcher.rs | 88 ++ rust/src/project/project.rs | 11 +- rust/src/project/project_api_impl.rs | 1061 +++++++++-------- rust/src/project/project_driver.rs | 144 +-- rust/src/project/sync_automerge_to_fs.rs | 139 +++ rust/src/project/sync_fs_to_automerge.rs | 195 ++++ 29 files changed, 4647 insertions(+), 1986 deletions(-) create mode 100644 rust/src/project/branch_db.rs create mode 100644 rust/src/project/branch_db/branch.rs create mode 100644 rust/src/project/branch_db/commit.rs create mode 100644 rust/src/project/branch_db/file.rs create mode 100644 rust/src/project/branch_db/util.rs create mode 100644 rust/src/project/connection.rs create mode 100644 rust/src/project/document_watcher.rs create mode 100644 rust/src/project/driver.rs create mode 100644 rust/src/project/fs_watcher.rs create mode 100644 rust/src/project/new_project.rs create mode 100644 rust/src/project/peer_watcher.rs create mode 100644 rust/src/project/sync_automerge_to_fs.rs create mode 100644 rust/src/project/sync_fs_to_automerge.rs diff --git a/Cargo.lock b/Cargo.lock index 652f1468..8a13de22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,28 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -1049,6 +1071,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "md5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" + [[package]] name = "memchr" version = "2.7.6" @@ -1222,6 +1250,7 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" name = "patchwork_rust" version = "0.0.1" dependencies = [ + "async-stream", "automerge", "autosurgeon", "cargo-post", @@ -1230,6 +1259,7 @@ dependencies = [ "futures", "glob", "godot", + "md5", "notify", "notify-debouncer-mini", "pretty_assertions", @@ -1245,6 +1275,7 @@ dependencies = [ "static_vcruntime", "tempfile", "tokio", + "tokio-stream", "tokio-tungstenite", "tracing", "tracing-appender", @@ -1252,7 +1283,6 @@ dependencies = [ "tree-sitter", "tree-sitter-godot-resource", "uuid", - "yet-another-md5", ] [[package]] @@ -1967,9 +1997,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ "bytes", "libc", @@ -1993,6 +2023,18 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-tungstenite" version = "0.27.0" @@ -2644,16 +2686,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" -[[package]] -name = "yet-another-md5" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bba9e0849c3974605f963b44cb18cb138184e4455f9533986ec35aa469c57ea7" -dependencies = [ - "log", - "thiserror 1.0.69", -] - [[package]] name = "zerocopy" version = "0.8.31" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 7c4f35df..7776b62b 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -24,7 +24,7 @@ automerge = "0.7.2" autosurgeon = "0.10.1" futures = "0.3.31" godot = "0.4.3" -tokio = { version = "1.41.1", features = ["full"] } +tokio = { version = "1.49.0", features = ["full"] } tracing-subscriber = { version = "0.3.19", features = ["env-filter", "json"] } tree-sitter = "0.25.4" tree-sitter-godot-resource = { git = "https://github.com/nikitalita/tree-sitter-godot-resource.git", branch = "fix-string-names" } @@ -47,7 +47,7 @@ rand = "0.9.2" # patchwork_rust_post_build_script = { path = "patchwork_rust_post_build_script" } serde = { version = "1.0", features = ["derive"] } -yet-another-md5 = "2.0.0" +md5 = "0.8.0" glob = "0.3.1" rlimit = "0.10.2" tracing = "0.1.40" @@ -57,6 +57,8 @@ samod = { version="0.7.1", features = ["tokio", "threadpool", "tungstenite"] } rayon = "1.11.0" regex = "1.12.2" tokio-tungstenite = "0.27.0" +async-stream = "0.3.6" +tokio-stream = { version = "0.1.18", features = ["sync"] } [build-dependencies] cbindgen = "^0.27" diff --git a/rust/src/diff/differ.rs b/rust/src/diff/differ.rs index 0c0c77a3..8843fccb 100644 --- a/rust/src/diff/differ.rs +++ b/rust/src/diff/differ.rs @@ -8,16 +8,19 @@ use automerge::ChangeHash; use godot::{ builtin::{GString, Variant}, classes::{ResourceLoader, resource_loader::CacheMode}, - meta::ToGodot, obj::Singleton, + meta::ToGodot, + obj::Singleton, }; use tracing::instrument; use crate::{ diff::{resource_differ::ResourceDiff, scene_differ::SceneDiff, text_differ::TextDiff}, - fs::{file_system_driver::FileSystemEvent, file_utils::FileContent}, + fs::{file_utils::FileSystemEvent, file_utils::FileContent}, helpers::{branch::BranchState, utils::ToShortForm}, - interop::{godot_accessors::PatchworkEditorAccessor}, - project::project::Project, + interop::godot_accessors::PatchworkEditorAccessor, + project::{ + branch_db::{BranchDb, HistoryRef}, + }, }; /// The type of change that occurred in a diff. @@ -50,81 +53,61 @@ pub struct ProjectDiff { } /// Computes diffs between two sets of heads in a project. -pub struct Differ<'a> { - /// The project we're diffing. - pub(super) project: &'a Project, - - /// The current heads we're diffing between. - pub(super) curr_heads: Vec, - - /// The previous heads we're diffing between. - pub(super) prev_heads: Vec, - +pub struct Differ { /// Cache that stores our loaded ExtResources so far. loaded_ext_resources: RefCell>, - // The branch we're currently diffing on - pub(super) branch_state: &'a BranchState, + /// The [BranchDb] we're working off. + branch_db: BranchDb, } -impl<'a> Differ<'a> { +impl Differ { /// Creates a new [Differ]. - pub fn new( - project: &'a Project, - curr_heads: Vec, - prev_heads: Vec, - branch_state: &'a BranchState, - ) -> Self { - let curr_heads = if curr_heads.len() == 0 { - branch_state.synced_heads.clone() - } else { - curr_heads - }; - + pub fn new(branch_db: BranchDb) -> Self { Self { - project, - curr_heads, - prev_heads, + branch_db, loaded_ext_resources: RefCell::new(HashMap::new()), - branch_state, } } /// Saves and imports a temp resource at a given path for the specified heads. - fn get_resource_at( + async fn get_resource_at_ref( &self, path: &String, file_content: &FileContent, - heads: &Vec, + ref_: &HistoryRef, ) -> Option { let import_path = format!("{}.import", path); - let mut import_file_content = self.get_file_at(&import_path, Some(heads)); - if import_file_content.is_none() { - // try at current heads - import_file_content = self.get_file_at(&import_path, None); - } - return self.create_temp_resource_from_content( - &path, - &file_content, - &heads, - import_file_content.as_ref(), - ); + let mut import_file_content = self.get_file_at_ref(&import_path, ref_).await; + // TODO (Lilith): Reimplement this using branchDB + // if import_file_content.is_none() { + // // try at current heads + // import_file_content = self.get_file_at_ref(&import_path, None); + // } + return self + .create_temp_resource_from_content( + &path, + &file_content, + &ref_.heads.first().to_short_form(), + import_file_content.as_ref(), + ) + .await; } /// Creates a temporary resource from file content at a given path. - pub(super) fn create_temp_resource_from_content( + pub(super) async fn create_temp_resource_from_content( &self, path: &str, content: &FileContent, - heads: &Vec, + temp_id: &String, import_file_content: Option<&FileContent>, ) -> Option { - let temp_dir = format!("res://.patchwork/temp_{}/", heads.first().to_short_form()); + let temp_dir = format!("res://.patchwork/temp_{}/", temp_id); let temp_path = path.replace("res://", &temp_dir); - if let Err(e) = FileContent::write_file_content( - &PathBuf::from(self.project.globalize_path(&temp_path)), - content, - ) { + if let Err(e) = content + .write(&PathBuf::from(self.branch_db.globalize_path(&temp_path))) + .await + { tracing::error!("error writing file to temp path: {:?}", e); return None; } @@ -137,10 +120,9 @@ impl<'a> Differ<'a> { import_file_content.replace(r#"uid=uid://[^\n]+"#, "uid=uid://"); // write the import file content to the temp path let import_file_path: String = format!("{}.import", temp_path); - let _ = FileContent::write_file_content( - &PathBuf::from(self.project.globalize_path(&import_file_path)), - &FileContent::String(import_file_content), - ); + let _ = FileContent::String(import_file_content).write(&PathBuf::from( + self.branch_db.globalize_path(&import_file_path), + )); let res = PatchworkEditorAccessor::import_and_load_resource(&temp_path); if res.is_nil() { @@ -162,25 +144,31 @@ impl<'a> Differ<'a> { } /// Gets the file content at a given path for the specified heads. - pub(super) fn get_file_at( + pub(super) async fn get_file_at_ref( &self, path: &String, - heads: Option<&Vec>, + ref_: &HistoryRef, ) -> Option { let mut ret: Option = None; { - let files = self - .project - .get_files_at(heads, Some(&HashSet::from_iter(vec![path.clone()]))); + let Some(files) = self + .branch_db + .get_files_at_ref(ref_, &HashSet::from_iter(vec![path.clone()])) + .await + else { + return None; + }; for file in files.into_iter() { if file.0 == *path { ret = Some(file.1); break; } else { - panic!( + tracing::error!( "Returned a file that didn't match the path!?!??!?!?!?!?!?!!? {:?} != {:?}", - file.0, path + file.0, + path ); + return None; } } } @@ -188,21 +176,24 @@ impl<'a> Differ<'a> { } /// Loads an ExtResource given a path, using a cache. - pub(super) fn load_ext_resource( + pub(super) async fn load_ext_resource( &self, path: &String, - heads: &Vec, + ref_: &HistoryRef, ) -> Option { if let Some(resource) = self.loaded_ext_resources.borrow().get(path) { return Some(resource.clone()); } - let resource_content = self.get_file_at(path, Some(heads)); + let resource_content = self.get_file_at_ref(path, ref_).await; let Some(resource_content) = resource_content else { return None; }; - let Some(resource) = self.get_resource_at(path, &resource_content, heads) else { + let Some(resource) = self + .get_resource_at_ref(path, &resource_content, ref_) + .await + else { return None; }; @@ -214,29 +205,23 @@ impl<'a> Differ<'a> { /// Computes the diff between the two sets of heads. #[instrument(skip_all, level = tracing::Level::DEBUG)] - pub fn get_diff(&self) -> ProjectDiff { - tracing::debug!( - "branch {:?}, getting changes between {} and {}", - self.branch_state.name, - self.prev_heads.to_short_form(), - self.curr_heads.to_short_form() - ); - - if self.prev_heads == self.curr_heads { + pub async fn get_diff(&self, before: &HistoryRef, after: &HistoryRef) -> ProjectDiff { + if before == after { tracing::debug!("no changes"); return ProjectDiff::default(); } - let mut diffs: Vec = vec![]; - // Get old and new content - let new_file_contents = self.project.get_changed_file_content_between( - None, - self.branch_state.doc_handle.document_id().clone(), - self.prev_heads.clone(), - self.curr_heads.clone(), - false, - ); - let changed_files_set: HashSet = new_file_contents + // Get the set of new file content that has changed + let Some(new_file_contents) = self + .branch_db + .get_changed_file_content_between_refs(Some(before), after, false) + .await + else { + // Something went wrong + return ProjectDiff::default(); + }; + + let changed_filter: HashSet = new_file_contents .iter() .map(|event| match event { FileSystemEvent::FileCreated(path, _) => path.to_string_lossy().to_string(), @@ -244,11 +229,18 @@ impl<'a> Differ<'a> { FileSystemEvent::FileDeleted(path) => path.to_string_lossy().to_string(), }) .collect::>(); - let old_file_contents = self.project.get_files_on_branch_at( - self.branch_state, - Some(&self.prev_heads), - Some(&changed_files_set), - ); + + // We do need to compare the new files to the old files, so grab the old contents with a filter + let Some(old_file_contents) = self + .branch_db + .get_files_at_ref(before, &changed_filter) + .await + else { + // Something went wrong + return ProjectDiff::default(); + }; + + let mut diffs: Vec = vec![]; for event in &new_file_contents { let (path, new_file_content, change_type) = match event { @@ -278,18 +270,24 @@ impl<'a> Differ<'a> { }; // This is a scene file, so use a scene diff diffs.push(Diff::Scene( - self.get_scene_diff(&path, old_scene, new_scene), + self.get_scene_diff(&path, old_scene, new_scene, before, after) + .await, )); } else if matches!(old_file_content, FileContent::Binary(_)) || matches!(new_file_content, FileContent::Binary(_)) { // This is a binary file, so use a resource diff - diffs.push(Diff::Resource(self.get_resource_diff( - &path, - change_type, - old_file_content, - new_file_content, - ))); + diffs.push(Diff::Resource( + self.get_resource_diff( + &path, + change_type, + old_file_content, + new_file_content, + before, + after, + ) + .await, + )); } else if matches!(old_file_content, FileContent::String(_)) || matches!(new_file_content, FileContent::String(_)) { diff --git a/rust/src/diff/resource_differ.rs b/rust/src/diff/resource_differ.rs index dc2e26a3..e540d19a 100644 --- a/rust/src/diff/resource_differ.rs +++ b/rust/src/diff/resource_differ.rs @@ -1,9 +1,7 @@ -use automerge::ChangeHash; use godot::builtin::Variant; use crate::{ - diff::differ::{ChangeType, Differ}, - fs::file_utils::FileContent, + diff::differ::{ChangeType, Differ}, fs::file_utils::FileContent, helpers::utils::ToShortForm, project::branch_db::HistoryRef }; #[derive(Clone, Debug)] @@ -30,42 +28,45 @@ impl ResourceDiff { } } -impl Differ<'_> { - pub(super) fn get_resource_diff( +impl Differ { + pub(super) async fn get_resource_diff( &self, path: &String, change_type: ChangeType, old_content: &FileContent, new_content: &FileContent, + before: &HistoryRef, + after: &HistoryRef ) -> ResourceDiff { ResourceDiff::new( path.clone(), change_type, - self.get_resource(path, old_content, &self.prev_heads), - self.get_resource(path, new_content, &self.curr_heads), + self.get_resource(path, old_content, before).await, + self.get_resource(path, new_content, after).await, ) } - fn get_resource( + async fn get_resource( &self, path: &String, content: &FileContent, - heads: &Vec, + ref_: &HistoryRef, ) -> Option { let import_path = format!("{}.import", path); let import_file_content = match content { FileContent::Deleted => None, _ => self - .get_file_at(&import_path, Some(heads)) - // try at current heads - .or(self.get_file_at(&import_path, None)), + .get_file_at_ref(&import_path, ref_).await + // TODO (Lilith): make this work + // try at current heads + // .or(self.get_file_at(&import_path, None)), }; self.create_temp_resource_from_content( &path, content, - &self.prev_heads, + &ref_.heads.first().to_short_form(), import_file_content.as_ref(), - ) + ).await } } diff --git a/rust/src/diff/scene_differ.rs b/rust/src/diff/scene_differ.rs index 3f8584da..9518f767 100644 --- a/rust/src/diff/scene_differ.rs +++ b/rust/src/diff/scene_differ.rs @@ -2,9 +2,10 @@ use std::collections::{HashMap, HashSet}; use godot::{ builtin::{StringName, Variant}, - classes::{ClassDb}, - global::str_to_var, - meta::ToGodot, obj::Singleton, + classes::ClassDb, + global::str_to_var, + meta::ToGodot, + obj::Singleton, }; use crate::{ @@ -12,6 +13,7 @@ use crate::{ parser::godot_parser::{ ExternalResourceNode, GodotNode, GodotScene, SubResourceNode, TypeOrInstance, }, + project::branch_db::HistoryRef, }; /// Represents a diff of a scene, with a scene path and a list of changed nodes. @@ -119,13 +121,15 @@ impl std::fmt::Display for VariantStrValue { } /// Implement scene-related functions on the Differ -impl Differ<'_> { +impl Differ { /// Generate a [SceneDiff] between the previous and current heads. - pub(super) fn get_scene_diff( + pub(super) async fn get_scene_diff( &self, path: &String, old_scene: Option<&GodotScene>, new_scene: Option<&GodotScene>, + before: &HistoryRef, + after: &HistoryRef, ) -> SceneDiff { let mut node_ids = HashSet::new(); let mut sub_resource_ids = HashSet::new(); @@ -156,7 +160,7 @@ impl Differ<'_> { let old_node = old_scene.as_ref().and_then(|s| s.get_node(*node_id)); let new_node = new_scene.as_ref().and_then(|s| s.get_node(*node_id)); - let Some(diff) = self.get_node_diff(*node_id, old_node, new_node, old_scene, new_scene) + let Some(diff) = self.get_node_diff(*node_id, old_node, new_node, old_scene, new_scene, before, after).await else { // If the node has no changes or is otherwise invalid, just skip this one. continue; @@ -177,13 +181,15 @@ impl Differ<'_> { } /// Generate a [NodeDiff] between two nodes. - fn get_node_diff( + async fn get_node_diff( &self, node_id: i32, old_node: Option<&GodotNode>, new_node: Option<&GodotNode>, old_scene: Option<&GodotScene>, new_scene: Option<&GodotScene>, + before: &HistoryRef, + after: &HistoryRef, ) -> Option { if old_node.is_none() && new_node.is_none() { return None; @@ -210,7 +216,7 @@ impl Differ<'_> { // Iterate through the props for prop in &props { if let Some(prop_diff) = - self.get_property_diff(prop, old_node, new_node, old_scene, new_scene) + self.get_property_diff(prop, old_node, new_node, old_scene, new_scene, before, after).await { changed_properties.insert(prop.clone(), prop_diff); } @@ -292,13 +298,15 @@ impl Differ<'_> { /// Returns a [PropertyDiff] comparing the old property value versus the new one. /// Returns [None] if neither node is valid, or if the value has not meaningfully changed. - fn get_property_diff( + async fn get_property_diff( &self, prop: &String, old_node: Option<&GodotNode>, new_node: Option<&GodotNode>, old_scene: Option<&GodotScene>, new_scene: Option<&GodotScene>, + before: &HistoryRef, + after: &HistoryRef, ) -> Option { // If neither node is valid, there's no valid property diff. if new_node.is_none() && old_node.is_none() { @@ -315,12 +323,14 @@ impl Differ<'_> { } // Expensive: Load any ext resources and turn them into Variants - let old = old_value - .as_ref() - .map(|v| self.get_prop_value(&v, old_scene, true, prop == "script")); - let new = new_value - .as_ref() - .map(|v| self.get_prop_value(&v, new_scene, false, prop == "script")); + let old = match &old_value { + Some(v) => Some(self.get_prop_value(v, old_scene, true, prop == "script", before, after).await), + None => None, + }; + let new = match &old_value { + Some(v) => Some(self.get_prop_value(v, new_scene, false, prop == "script", before, after).await), + None => None, + }; return Some(PropertyDiff::new( prop.clone(), @@ -461,12 +471,14 @@ impl Differ<'_> { /// Returns the value of a given prop, within a given scene. /// Normally, it's a String. If it's a (non-script) ExtResource or ResourcePath, /// it loads and returns the resource content as a Variant. - fn get_prop_value( + async fn get_prop_value( &self, prop_value: &VariantStrValue, scene: Option<&GodotScene>, is_old: bool, is_script: bool, + before: &HistoryRef, + after: &HistoryRef, ) -> Variant { // Prevent loading script files during the diff and creating issues for the editor if is_script { @@ -498,14 +510,8 @@ impl Differ<'_> { } } - let Some(resource) = self.load_ext_resource( - &path, - if is_old { - &self.prev_heads - } else { - &self.curr_heads - }, - ) else { + let Some(resource) = self.load_ext_resource(&path, if is_old { before } else { after }).await + else { return "".to_variant(); }; diff --git a/rust/src/diff/text_differ.rs b/rust/src/diff/text_differ.rs index 525507e7..e91b810b 100644 --- a/rust/src/diff/text_differ.rs +++ b/rust/src/diff/text_differ.rs @@ -94,7 +94,7 @@ impl TextDiff { } } -impl Differ<'_> { +impl Differ { pub(super) fn get_text_diff( &self, path: &String, diff --git a/rust/src/fs.rs b/rust/src/fs.rs index 5573dbe0..6c7abba2 100644 --- a/rust/src/fs.rs +++ b/rust/src/fs.rs @@ -1,2 +1,2 @@ -pub mod file_system_driver; -pub mod file_utils; +// pub mod file_system_driver; +pub mod file_utils; \ No newline at end of file diff --git a/rust/src/fs/file_system_driver.rs b/rust/src/fs/file_system_driver.rs index 9e1825a8..63f62bb1 100644 --- a/rust/src/fs/file_system_driver.rs +++ b/rust/src/fs/file_system_driver.rs @@ -1,107 +1,122 @@ +use crate::helpers::utils::ToShortForm; use core::str; -use std::sync::atomic::Ordering; -use std::{path::PathBuf, sync::atomic::AtomicBool}; -use std::sync::Arc; +use std::path::Path; use futures::Stream; +use futures::stream::FuturesUnordered; use futures::{ - channel::mpsc::{UnboundedReceiver, UnboundedSender}, StreamExt, + channel::mpsc::{UnboundedReceiver, UnboundedSender}, }; +use glob::Pattern; +use notify::RecommendedWatcher; +use notify::{Config, RecursiveMode}; +use notify_debouncer_mini::{DebouncedEvent, Debouncer, new_debouncer_opt}; #[cfg(not(target_os = "windows"))] -use rlimit::{setrlimit, getrlimit, Resource}; -use tokio::{task::JoinHandle, time::{sleep, Duration}}; -use notify::{RecursiveMode, Config}; -use notify_debouncer_mini::{new_debouncer_opt, DebouncedEvent, Debouncer}; -use notify::RecommendedWatcher as WatcherImpl; -use tracing::instrument; -use std::time::Duration as StdDuration; +use rlimit::{Resource, getrlimit, setrlimit}; +use tokio::sync::mpsc::{Receiver, channel}; use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Duration as StdDuration; +use std::{path::PathBuf, sync::atomic::AtomicBool}; use tokio::sync::Mutex; -use glob::Pattern; -use crate::helpers::utils::ToShortForm; +use tokio::{ + task::JoinHandle, + time::{Duration, sleep}, +}; +use tracing::instrument; +use notify::{Event, RecommendedWatcher, Watcher}; -use super::file_utils::{calculate_file_hash, get_buffer_and_hash, FileContent}; +use super::file_utils::{FileContent, calculate_file_hash, get_buffer_and_hash}; -#[cfg(test)] -mod tests; +// #[cfg(test)] +// mod tests; // static const var for debounce time const DEBOUNCE_TIME: u64 = 100; -#[derive(Debug)] -pub enum FileSystemEvent { - FileCreated(PathBuf, FileContent), - FileModified(PathBuf, FileContent), - FileDeleted(PathBuf), -} - impl ToShortForm for FileSystemEvent { - fn to_short_form(&self) -> String { - let content_type = match self { - FileSystemEvent::FileCreated(_, content) => match content { - FileContent::Scene(_) => "scene", - FileContent::String(_) => "text", - FileContent::Binary(_) => "binary", - FileContent::Deleted => "deleted", - }, - FileSystemEvent::FileModified(_, content) => match content { - FileContent::Scene(_) => "scene", - FileContent::String(_) => "text", - FileContent::Binary(_) => "binary", - FileContent::Deleted => "deleted", - }, - _ => "deleted", - }; - match self { - FileSystemEvent::FileCreated(path, _) => format!("FileCreated({:?}, {})", path, content_type), - FileSystemEvent::FileModified(path_buf, _) => format!("FileModified({:?}, {})", path_buf, content_type), - FileSystemEvent::FileDeleted(path_buf) => format!("FileDeleted({:?}, {})", path_buf, content_type), - } - } + fn to_short_form(&self) -> String { + let content_type = match self { + FileSystemEvent::FileCreated(_, content) => match content { + FileContent::Scene(_) => "scene", + FileContent::String(_) => "text", + FileContent::Binary(_) => "binary", + FileContent::Deleted => "deleted", + }, + FileSystemEvent::FileModified(_, content) => match content { + FileContent::Scene(_) => "scene", + FileContent::String(_) => "text", + FileContent::Binary(_) => "binary", + FileContent::Deleted => "deleted", + }, + _ => "deleted", + }; + match self { + FileSystemEvent::FileCreated(path, _) => { + format!("FileCreated({:?}, {})", path, content_type) + } + FileSystemEvent::FileModified(path_buf, _) => { + format!("FileModified({:?}, {})", path_buf, content_type) + } + FileSystemEvent::FileDeleted(path_buf) => { + format!("FileDeleted({:?}, {})", path_buf, content_type) + } + } + } } impl ToShortForm for Vec { - fn to_short_form(&self) -> String { - self.iter().map(|e| e.to_short_form()).collect::>().join(", ") - } + fn to_short_form(&self) -> String { + self.iter() + .map(|e| e.to_short_form()) + .collect::>() + .join(", ") + } } - #[derive(Debug)] pub enum FileSystemUpdateEvent { FileSaved(PathBuf, FileContent), FileDeleted(PathBuf), - Pause, - Resume + Pause, + Resume, } impl ToShortForm for FileSystemUpdateEvent { - fn to_short_form(&self) -> String { - let content_type = match self { - FileSystemUpdateEvent::FileSaved(_, content) => match content { - FileContent::Scene(_) => "scene", - FileContent::String(_) => "text", - FileContent::Binary(_) => "binary", - FileContent::Deleted => "deleted", - }, - FileSystemUpdateEvent::FileDeleted(_) => "deleted", - FileSystemUpdateEvent::Pause => "", - FileSystemUpdateEvent::Resume => "", - }; - - match self { - FileSystemUpdateEvent::FileSaved(path, _) => format!("FileSaved({:?} {})", path, content_type), - FileSystemUpdateEvent::FileDeleted(path) => format!("FileDeleted({:?} {})", path, content_type), - FileSystemUpdateEvent::Pause => "Pause".to_string(), - FileSystemUpdateEvent::Resume => "Resume".to_string(), - } - } + fn to_short_form(&self) -> String { + let content_type = match self { + FileSystemUpdateEvent::FileSaved(_, content) => match content { + FileContent::Scene(_) => "scene", + FileContent::String(_) => "text", + FileContent::Binary(_) => "binary", + FileContent::Deleted => "deleted", + }, + FileSystemUpdateEvent::FileDeleted(_) => "deleted", + FileSystemUpdateEvent::Pause => "", + FileSystemUpdateEvent::Resume => "", + }; + + match self { + FileSystemUpdateEvent::FileSaved(path, _) => { + format!("FileSaved({:?} {})", path, content_type) + } + FileSystemUpdateEvent::FileDeleted(path) => { + format!("FileDeleted({:?} {})", path, content_type) + } + FileSystemUpdateEvent::Pause => "Pause".to_string(), + FileSystemUpdateEvent::Resume => "Resume".to_string(), + } + } } impl ToShortForm for Vec { - fn to_short_form(&self) -> String { - self.iter().map(|e| e.to_short_form()).collect::>().join(", ") - } + fn to_short_form(&self) -> String { + self.iter() + .map(|e| e.to_short_form()) + .collect::>() + .join(", ") + } } #[derive(Debug, Clone)] @@ -109,73 +124,72 @@ pub struct FileSystemTask { watch_path: PathBuf, file_hashes: Arc>>, ignore_globs: Vec, - watcher: Arc>>, - // atomic bool - paused: Arc, - found_ignored_paths: HashSet, + watcher: Arc>>, + // atomic bool + paused: Arc, + found_ignored_paths: Arc>>, } #[derive(Debug)] pub struct FileSystemDriver { - task: FileSystemTask, - output_rx: UnboundedReceiver, - input_tx: UnboundedSender, - handle: JoinHandle<()>, - rt: Option, + task: FileSystemTask, + output_rx: UnboundedReceiver, + input_tx: UnboundedSender, + handle: JoinHandle<()>, + rt: Option, } impl FileSystemTask { // Check if a path should be ignored based on glob patterns fn should_ignore(&self, path: &PathBuf) -> bool { - // TODO: We should check if it's a symlink or not, but right now it's sufficient to just check if it's outside of the watch path - // check if it's outside of the watch path - if path.is_symlink() { - return true; - } - if !path.starts_with(&self.watch_path) { - return true; - } + // TODO: We should check if it's a symlink or not, but right now it's sufficient to just check if it's outside of the watch path + // check if it's outside of the watch path + if path.is_symlink() { + return true; + } + if !path.starts_with(&self.watch_path) { + return true; + } let path_str = path.to_string_lossy(); - self.ignore_globs.iter().any(|pattern| pattern.matches(&path_str)) + self.ignore_globs + .iter() + .any(|pattern| pattern.matches(&path_str)) } + async fn initialize_file_hashes_recur(&self, watch_path: PathBuf) -> tokio::io::Result<()> { + let mut dir = tokio::fs::read_dir(watch_path).await?; + let mut sub_tasks = FuturesUnordered::new(); + while let Some(entry) = dir.next_entry().await? { + let path = entry.path(); + + // Skip if path matches any ignore pattern + if self.should_ignore(&path) { + let mut found_ignored_paths = self.found_ignored_paths.lock().await; + found_ignored_paths.insert(path.clone()); + continue; + } - fn _initialize_file_hashes(&self, watch_path: &PathBuf, file_hashes: &mut tokio::sync::MutexGuard<'_, HashMap>, sym_links: &mut HashMap, found_ignored_paths: &mut HashSet) { - if let Ok(entries) = std::fs::read_dir(watch_path) { - for entry in entries.flatten() { - let path = entry.path(); - - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - found_ignored_paths.insert(path.clone()); - continue; - } - if let Ok(metadata) = path.metadata() { - if metadata.is_symlink() { - let target = std::fs::read_link(&path).unwrap(); - sym_links.insert(path.clone(), target); - } - } - - if path.is_file() { - // check if the path is a symlink - if let Some(hash) = calculate_file_hash(&path) { - file_hashes.insert(path, hash); - } - } else if path.is_dir() { - self._initialize_file_hashes(&path, file_hashes, sym_links, found_ignored_paths); + if path.is_file() { + if let Some(hash) = calculate_file_hash(&path) { + let mut file_hashes = self.file_hashes.lock().await; + file_hashes.insert(path, hash); } + } else if path.is_dir() { + let path = path.clone(); + sub_tasks.push(self.initialize_file_hashes_recur(path)); } } + while let Some(_) = sub_tasks.next().await {} + Ok(()) } // Initialize the hash map with existing files - async fn initialize_file_hashes(&mut self, sym_links: &mut HashMap) { - let mut found_ignored_paths = HashSet::new(); - let mut file_hashes_guard: tokio::sync::MutexGuard<'_, HashMap> = self.file_hashes.lock().await; - file_hashes_guard.clear(); - self._initialize_file_hashes(&self.watch_path, &mut file_hashes_guard, sym_links, &mut found_ignored_paths); - self.found_ignored_paths = found_ignored_paths; + async fn initialize_file_hashes(&self) { + { + self.file_hashes.lock().await.clear(); + self.found_ignored_paths.lock().await.clear(); + } + self.initialize_file_hashes_recur(self.watch_path.clone()); } // Handle file creation and modification events @@ -187,603 +201,554 @@ impl FileSystemTask { if self.should_ignore(&path) { return Ok(None); } - if !path.exists() { - // If the file doesn't exist, we want to emit a deleted event - let mut file_hashes: tokio::sync::MutexGuard<'_, HashMap> = self.file_hashes.lock().await; - if file_hashes.contains_key(&path) { - file_hashes.remove(&path); - return Ok(Some(FileSystemEvent::FileDeleted(path))); - } - return Ok(None); - } + if !path.exists() { + // If the file doesn't exist, we want to emit a deleted event + let mut file_hashes: tokio::sync::MutexGuard<'_, HashMap> = + self.file_hashes.lock().await; + if file_hashes.contains_key(&path) { + file_hashes.remove(&path); + return Ok(Some(FileSystemEvent::FileDeleted(path))); + } + return Ok(None); + } if path.is_file() { - let mut result = get_buffer_and_hash(&path); - if result.is_err() { - sleep(StdDuration::from_millis(DEBOUNCE_TIME)).await; - result = get_buffer_and_hash(&path); - } - if result.is_err() { - tracing::error!("failed to get file content {:?}", result); - return Err(notify::Error::new(notify::ErrorKind::Generic("Failed to get file content".to_string()))); - } - let (content, new_hash) = result.unwrap(); - let mut file_hashes: tokio::sync::MutexGuard<'_, HashMap> = self.file_hashes.lock().await; - if file_hashes.contains_key(&path) { - let old_hash = file_hashes.get(&path).unwrap(); - if old_hash != &new_hash { - tracing::trace!("file {:?} changed, hash {} -> {}", path, old_hash, new_hash); - file_hashes.insert(path.clone(), new_hash); - return Ok(Some(FileSystemEvent::FileModified(path, FileContent::from_buf(content)))); - } - } else { - // If the file is newly created, we want to emit a created event - tracing::trace!("file {:?} created, hash {}", path, new_hash); - file_hashes.insert(path.clone(), new_hash); - return Ok(Some(FileSystemEvent::FileCreated(path, FileContent::from_buf(content)))); - } + let mut result = get_buffer_and_hash(&path); + if result.is_err() { + sleep(StdDuration::from_millis(DEBOUNCE_TIME)).await; + result = get_buffer_and_hash(&path); + } + if result.is_err() { + tracing::error!("failed to get file content {:?}", result); + return Err(notify::Error::new(notify::ErrorKind::Generic( + "Failed to get file content".to_string(), + ))); + } + let (content, new_hash) = result.unwrap(); + let mut file_hashes: tokio::sync::MutexGuard<'_, HashMap> = + self.file_hashes.lock().await; + if file_hashes.contains_key(&path) { + let old_hash = file_hashes.get(&path).unwrap(); + if old_hash != &new_hash { + tracing::trace!("file {:?} changed, hash {} -> {}", path, old_hash, new_hash); + file_hashes.insert(path.clone(), new_hash); + return Ok(Some(FileSystemEvent::FileModified( + path, + FileContent::from_buf(content), + ))); + } + } else { + // If the file is newly created, we want to emit a created event + tracing::trace!("file {:?} created, hash {}", path, new_hash); + file_hashes.insert(path.clone(), new_hash); + return Ok(Some(FileSystemEvent::FileCreated( + path, + FileContent::from_buf(content), + ))); + } } - Ok(None) + Ok(None) } - // handle syncs from patchwork - pub async fn handle_file_update( + // handle syncs from patchwork + async fn handle_file_update( &self, path: PathBuf, - content: FileContent, + content: FileContent, ) -> Result<(), notify::Error> { - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - return Ok(()); - } - - // Write the file content to disk - if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { - let mut file_hashes = self.file_hashes.lock().await; - file_hashes.insert(path.clone(), hash_str); - } else { - return Err(notify::Error::new(notify::ErrorKind::Generic("Failed to write file".to_string()))); - } - Ok(()) - } - - pub fn handle_file_update_blocking( - &self, - path: PathBuf, - content: FileContent, - ) -> Result<(), notify::Error> { - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - return Ok(()); - } + // Skip if path matches any ignore pattern + if self.should_ignore(&path) { + return Ok(()); + } - // Write the file content to disk - if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { - let mut file_hashes = self.file_hashes.blocking_lock(); - file_hashes.insert(path.clone(), hash_str); - } else { - return Err(notify::Error::new(notify::ErrorKind::Generic("Failed to write file".to_string()))); - } - Ok(()) + // Write the file content to disk + if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { + let mut file_hashes = self.file_hashes.lock().await; + file_hashes.insert(path.clone(), hash_str); + } else { + return Err(notify::Error::new(notify::ErrorKind::Generic( + "Failed to write file".to_string(), + ))); + } + Ok(()) } - - pub async fn handle_delete_update( - &self, - path: PathBuf, - ) -> Result<(), notify::Error> { + pub async fn handle_delete_update(&self, path: PathBuf) -> Result<(), notify::Error> { // Skip if path matches any ignore pattern if self.should_ignore(&path) { return Ok(()); } - // Delete the file from disk - if std::fs::remove_file(&path.canonicalize().unwrap()).is_ok() { - // Remove the hash from our tracking - let mut file_hashes = self.file_hashes.lock().await; - file_hashes.remove(&path); - - } else { - return Err(notify::Error::new(notify::ErrorKind::Generic("Failed to delete file".to_string()))); + // Delete the file from disk + if std::fs::remove_file(&path.canonicalize().unwrap()).is_ok() { + // Remove the hash from our tracking + let mut file_hashes = self.file_hashes.lock().await; + file_hashes.remove(&path); + } else { + return Err(notify::Error::new(notify::ErrorKind::Generic( + "Failed to delete file".to_string(), + ))); } Ok(()) } - pub fn handle_delete_update_blocking( - &self, - path: PathBuf, - ) -> Result<(), notify::Error> { - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - return Ok(()); - } - - // Delete the file from disk - if std::fs::remove_file(&path.canonicalize().unwrap()).is_ok() { - // Remove the hash from our tracking - let mut file_hashes = self.file_hashes.blocking_lock(); - file_hashes.remove(&path); - } else { - return Err(notify::Error::new(notify::ErrorKind::Generic("Failed to delete file".to_string()))); - } - Ok(()) - } - - // Scan for changes in the file system - async fn _scan_for_additive_changes( - &self, - watch_path: &PathBuf, - ) -> Vec - { - let mut events = Vec::new(); - let entries = std::fs::read_dir(watch_path); - if entries.is_err() { - return events; - } - let entries = entries.unwrap(); - for entry in entries.flatten() { - let path = entry.path(); - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - continue; - } - - if path.is_file() { - let res = self.handle_file_event(path).await; - if let Ok(Some(ret)) = res{ - events.push(ret); - } - } else if path.is_dir() { - // Use Box::pin for the recursive call to avoid infinitely sized future - let sub_events = Box::pin(self._scan_for_additive_changes(&path)).await; - events.extend(sub_events); - } - } - events - } - - async fn scan_for_changes(&self) -> Vec { - let mut events = self._scan_for_additive_changes(&self.watch_path).await; - // check the file_hashes for removed files - let mut to_remove = Vec::new(); - let mut file_hashes = self.file_hashes.lock().await; - for (path, _) in file_hashes.iter() { - if !path.exists() { - to_remove.push(path.clone()); - } - } - for path in to_remove { - file_hashes.remove(&path); - events.push(FileSystemEvent::FileDeleted(path)); - } - events - } - - async fn process_notify_events(&mut self, notify_event: Vec, output_tx: &UnboundedSender) { - for event in notify_event { - if self.found_ignored_paths.contains(&event.path) { - continue; - } - if self.should_ignore(&event.path) { - self.found_ignored_paths.insert(event.path); - continue; - } - let result = self.handle_file_event(event.path.clone()).await; - if let Ok(Some(ret)) = result { - output_tx.unbounded_send(ret).ok(); - } - } - } - - - async fn main_loop(&mut self, notify_rx: &mut UnboundedReceiver, notify::Error>>, input_rx: &mut UnboundedReceiver, output_tx: &UnboundedSender) { - let mut sym_links = HashMap::new(); - self.initialize_file_hashes(&mut sym_links).await; - self.stop_watching_paths(&self.found_ignored_paths).await; - // Process both file system events and update events - loop { - tokio::select! { - // Handle file system events - Some(notify_result) = notify_rx.next() => { - if let Ok(notify_event) = notify_result { - self.process_notify_events(notify_event, output_tx).await; - } - }, - // Handle update events - Some(event) = input_rx.next() => { - match event { - FileSystemUpdateEvent::FileSaved(path, content) => { - let result = self.handle_file_update(path, content).await; - if result.is_err() { - tracing::error!("failed to handle file update {:?}", result); - } - } - FileSystemUpdateEvent::FileDeleted(path) => { - let result = self.handle_delete_update(path).await; - if result.is_err() { - tracing::error!("failed to handle file delete {:?}", result); - } - } - FileSystemUpdateEvent::Pause => { - self.stop_watching_path(&self.watch_path).await; - self.pause(); - } - FileSystemUpdateEvent::Resume => { - self.start_watching_path(&self.watch_path).await; - self.stop_watching_paths(&self.found_ignored_paths).await; - self.resume(); - // let events = self.scan_for_changes(&mut sym_links).await; - // for event in events { - // output_tx.unbounded_send(event).ok(); - // } - } - } - }, - } - } - } - - async fn stop_watching_paths(&self, paths: &HashSet) { - let mut watcher = self.watcher.lock().await; - for path in paths.iter() { - let _ret = watcher.watcher().unwatch(path); - } - } - - async fn stop_watching_path(&self, path: &PathBuf) { - let _ = self.watcher.lock().await.watcher().unwatch(path); - } - - fn stop_watching_path_blocking(&self, path: &PathBuf) { - let _ = self.watcher.blocking_lock().watcher().unwatch(path); - } - - async fn start_watching_path(&self, path: &PathBuf) { - let _ = self.watcher.lock().await.watcher().watch(path, RecursiveMode::Recursive); - } - - fn start_watching_path_blocking(&self, path: &PathBuf) { - let _ = self.watcher.blocking_lock().watcher().watch(path, RecursiveMode::Recursive); - } - - async fn add_ignore_glob(&mut self, glob: &str) { - self.ignore_globs.push(Pattern::new(glob).unwrap()); - } - pub fn is_paused(&self) -> bool { - self.paused.load(Ordering::Relaxed) - } - - - fn pause(&self) { - self.paused.store(true, Ordering::Relaxed); - } - - fn resume(&self) { - self.paused.store(false, Ordering::Relaxed); - } + // Scan for changes in the file system + async fn scan_for_additive_changes(&self, watch_path: &PathBuf) -> Vec { + let mut events = Vec::new(); + let entries = std::fs::read_dir(watch_path); + if entries.is_err() { + return events; + } + let entries = entries.unwrap(); + for entry in entries.flatten() { + let path = entry.path(); + // Skip if path matches any ignore pattern + if self.should_ignore(&path) { + continue; + } + + if path.is_file() { + let res = self.handle_file_event(path).await; + if let Ok(Some(ret)) = res { + events.push(ret); + } + } else if path.is_dir() { + // Use Box::pin for the recursive call to avoid infinitely sized future + let sub_events = Box::pin(self.scan_for_additive_changes(&path)).await; + events.extend(sub_events); + } + } + events + } + + async fn process_notify_events( + &mut self, + notify_event: Vec, + output_tx: &UnboundedSender, + ) { + for event in notify_event { + if self.found_ignored_paths.contains(&event.path) { + continue; + } + if self.should_ignore(&event.path) { + self.found_ignored_paths.insert(event.path); + continue; + } + let result = self.handle_file_event(event.path.clone()).await; + if let Ok(Some(ret)) = result { + output_tx.unbounded_send(ret).ok(); + } + } + } + + async fn main_loop( + &mut self, + notify_rx: &mut UnboundedReceiver, notify::Error>>, + input_rx: &mut UnboundedReceiver, + output_tx: &UnboundedSender, + ) { + let mut sym_links = HashMap::new(); + self.initialize_file_hashes().await; + self.stop_watching_paths(&self.found_ignored_paths).await; + // Process both file system events and update events + loop { + tokio::select! { + // Handle file system events + Some(notify_result) = notify_rx.next() => { + if let Ok(notify_event) = notify_result { + self.process_notify_events(notify_event, output_tx).await; + } + }, + // Handle update events + Some(event) = input_rx.next() => { + match event { + FileSystemUpdateEvent::FileSaved(path, content) => { + let result = self.handle_file_update(path, content).await; + if result.is_err() { + tracing::error!("failed to handle file update {:?}", result); + } + } + FileSystemUpdateEvent::FileDeleted(path) => { + let result = self.handle_delete_update(path).await; + if result.is_err() { + tracing::error!("failed to handle file delete {:?}", result); + } + } + FileSystemUpdateEvent::Pause => { + self.stop_watching_path(&self.watch_path).await; + self.pause(); + } + FileSystemUpdateEvent::Resume => { + self.start_watching_path(&self.watch_path).await; + self.stop_watching_paths(&self.found_ignored_paths).await; + self.resume(); + // let events = self.scan_for_changes(&mut sym_links).await; + // for event in events { + // output_tx.unbounded_send(event).ok(); + // } + } + } + }, + } + } + } + async fn stop_watching_paths(&self, paths: &HashSet) { + let mut watcher = self.watcher.lock().await; + for path in paths.iter() { + let _ret = watcher.watcher().unwatch(path); + } + } + + async fn stop_watching_path(&self, path: &PathBuf) { + let _ = self.watcher.lock().await.watcher().unwatch(path); + } + + async fn start_watching_path(&self, path: &PathBuf) { + let _ = self + .watcher + .lock() + .await + .watcher() + .watch(path, RecursiveMode::Recursive); + } + + pub fn is_paused(&self) -> bool { + self.paused.load(Ordering::Relaxed) + } + + fn pause(&self) { + self.paused.store(true, Ordering::Relaxed); + } + + fn resume(&self) { + self.paused.store(false, Ordering::Relaxed); + } } const MAX_OPEN_FILES: u64 = 100000000; - impl FileSystemDriver { - fn increase_ulimit() { - #[cfg(not(target_os = "windows"))] - { - let mut new_soft_limit = MAX_OPEN_FILES; - let mut new_hard_limit = MAX_OPEN_FILES; - let previous_result = getrlimit(Resource::NOFILE); - if let Err(e) = previous_result { - tracing::error!("failed to get ulimit {:?}", e); - } else if let Ok((soft_limit, hard_limit)) = previous_result { - tracing::debug!("soft ulimit {:?}", soft_limit); - tracing::debug!("hard ulimit {:?}", hard_limit); - if hard_limit > MAX_OPEN_FILES { - new_hard_limit = hard_limit; - } - if soft_limit > MAX_OPEN_FILES { - new_soft_limit = soft_limit; - } - } - - if let Err(e) = setrlimit(Resource::NOFILE, new_soft_limit, new_hard_limit) { - tracing::error!("failed to set ulimit {:?}", e); - } - let result = getrlimit(Resource::NOFILE); - if let Err(e) = result { - tracing::error!("failed to set ulimit {:?}", e); - } else if let Ok((soft_limit, hard_limit)) = result { - if soft_limit < MAX_OPEN_FILES || hard_limit < MAX_OPEN_FILES { - tracing::error!("failed to set ulimit; soft ulimit {:?}, hard ulimit {:?}", soft_limit, hard_limit); - } - } - } - } - - fn spawn_with_runtime(watch_path: PathBuf, ignore_globs: Vec, rt: Option) -> Self { - // if macos, increase ulimit to 100000000 - Self::increase_ulimit(); - let (output_tx, output_rx) = futures::channel::mpsc::unbounded(); - let (input_tx, mut input_rx) = futures::channel::mpsc::unbounded(); - // Spawn the file system watcher in a separate task - let notify_config = Config::default().with_follow_symlinks(false).with_ignore_globs(ignore_globs.clone()); - let ignore_globs: Vec = ignore_globs - .into_iter() - .filter_map(|glob_str| Pattern::new(&glob_str).ok()) - .collect(); - let (notify_tx, mut notify_rx) = futures::channel::mpsc::unbounded(); - - let debouncer_config = notify_debouncer_mini::Config::default().with_timeout(Duration::from_millis(DEBOUNCE_TIME)).with_batch_mode(true).with_notify_config(notify_config); - - let mut debouncer = new_debouncer_opt::<_, WatcherImpl>(debouncer_config, move |event: Result, notify::Error>| { - notify_tx.unbounded_send(event).unwrap(); - }).unwrap(); - debouncer.watcher().watch(&watch_path, RecursiveMode::Recursive).unwrap(); - - let rt_handle = if rt.is_some() { - rt.as_ref().unwrap().handle().clone() - } else { - tokio::runtime::Handle::current() - }; - let task: FileSystemTask = FileSystemTask { - watch_path: watch_path.clone(), - file_hashes: Arc::new(Mutex::new(HashMap::new())), - ignore_globs: ignore_globs, - watcher: Arc::new(Mutex::new(debouncer)), - paused: Arc::new(AtomicBool::new(false)), - found_ignored_paths: HashSet::new() - }; - - let mut this_task = task.clone(); - - let handle = rt_handle.spawn(async move { - this_task.main_loop(&mut notify_rx, &mut input_rx, &output_tx).await; - }); - Self { - task, - output_rx, - input_tx, - handle, - rt: rt - } - } - - pub fn spawn(watch_path: PathBuf, ignore_globs: Vec) -> Self { - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(1) - .thread_name("FileSystemDriver: watcher thread") - .enable_all() - .build() - .unwrap(); - Self::spawn_with_runtime(watch_path, ignore_globs, Some(rt)) - } - - - - pub fn save_file_async(&self, path: PathBuf, content: FileContent) { - self.input_tx.unbounded_send(FileSystemUpdateEvent::FileSaved(path, content)).ok(); - } - - pub fn delete_file_async(&self, path: PathBuf) { - self.input_tx.unbounded_send(FileSystemUpdateEvent::FileDeleted(path)).ok(); - } - - fn get_existing_parent_path(path: &PathBuf) -> Option { - let mut parent = Some(path.as_path()); - while parent.is_some() { - if parent.as_ref().unwrap().exists() { - break; - } - parent = parent.unwrap().parent(); - } - if parent.is_some() { - return Some(parent.unwrap().to_path_buf()); - } - None - } - - pub async fn save_file(&self, path: PathBuf, content: FileContent) -> Result<(), notify::Error> { - let parent = Self::get_existing_parent_path(&path); - if parent.is_some() { - let parent = parent.unwrap(); - self.task.stop_watching_path(&parent).await; - let result = self.task.handle_file_update(path.clone(), content).await; - self.task.start_watching_path(&parent).await; - return result; - } else { - let result = self.task.handle_file_update(path.clone(), content).await; - return result; - } - } - - pub fn save_file_blocking(&self, path: PathBuf, content: FileContent) -> Result<(), notify::Error> { - let parent = Self::get_existing_parent_path(&path); - if parent.is_some() { - let parent = parent.unwrap(); - self.task.stop_watching_path_blocking(&parent); - let result = self.task.handle_file_update_blocking(path.clone(), content); - self.task.start_watching_path_blocking(&parent); - return result; - } else { - let result = self.task.handle_file_update_blocking(path.clone(), content); - return result; - } - } - - async fn pause_task(&self) { - self.input_tx.unbounded_send(FileSystemUpdateEvent::Pause).ok(); - while !self.task.is_paused() { - sleep(Duration::from_millis(100)).await; - } - } - - fn pause_task_blocking(&self) { - self.input_tx.unbounded_send(FileSystemUpdateEvent::Pause).ok(); - while !self.task.is_paused() { - std::thread::sleep(Duration::from_millis(100)); - } - } - - async fn resume_task(&self) { - self.input_tx.unbounded_send(FileSystemUpdateEvent::Resume).ok(); - while self.task.is_paused() { - sleep(Duration::from_millis(100)).await; - } - } - - fn resume_task_blocking(&self) { - self.input_tx.unbounded_send(FileSystemUpdateEvent::Resume).ok(); - while self.task.is_paused() { - std::thread::sleep(Duration::from_millis(100)); - } - } - - pub async fn batch_update(&self, updates: Vec) { - self.pause_task().await; - { - let mut file_hashes = self.task.file_hashes.lock().await; - for update in updates { - match update { - FileSystemUpdateEvent::FileSaved(path, content) => { - if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { - file_hashes.insert(path.clone(), hash_str); - } else { - tracing::error!("failed to write file {:?}", path); - } - } - FileSystemUpdateEvent::FileDeleted(path) => { - file_hashes.remove(&path); - } - FileSystemUpdateEvent::Pause => { - continue; - } - FileSystemUpdateEvent::Resume => { - continue; - } - } - } - } - self.resume_task().await; - } - - #[instrument(skip_all, level = tracing::Level::INFO)] - pub fn batch_update_blocking(&self, updates: Vec) -> Vec { - tracing::debug!("# of updates: {:?}", updates.len()); - tracing::trace!("updates: [{}]", updates.to_short_form()); - self.pause_task_blocking(); - tracing::trace!("batch_update_blocking after pause"); - let mut events: Vec = Vec::new(); - { - let mut file_hashes = self.task.file_hashes.blocking_lock(); - for update in updates { - match update { - FileSystemUpdateEvent::FileSaved(path, content) => { - let new_hash_str = content.to_hash(); - let mut modified = false; - let mut created = false; - if let Some(old_hash) = file_hashes.get(&path) { - if old_hash != &new_hash_str { - modified = true; - } - } else { - created = true; - } - if modified || created { - if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { - if new_hash_str != hash_str { - tracing::error!("THIS SHOULD NOT HAPPEN: file {:?} previous calced hash {:?} != written hash {:?}", path, new_hash_str, hash_str); - } - if modified { - tracing::trace!("file {:?} changed, hash {} -> {}", path, file_hashes.get(&path).unwrap(), new_hash_str); - events.push(FileSystemEvent::FileModified(path.clone(), content)); - } else { - tracing::trace!("file {:?} created, hash {}", path, new_hash_str); - events.push(FileSystemEvent::FileCreated(path.clone(), content)); - } - file_hashes.insert(path, hash_str); - } else { - tracing::error!("failed to write file {:?}", path); - } - } else { - tracing::debug!("file {:?} already exists with same hash {:?}", path, new_hash_str); - } - } - FileSystemUpdateEvent::FileDeleted(path) => { - let _ = std::fs::remove_file(&path); - if file_hashes.remove(&path).is_some() { - events.push(FileSystemEvent::FileDeleted(path)); - } - } - _ => { - continue; - } - } - } - } - tracing::trace!("batch_update_blocking done, before resume"); - self.resume_task_blocking(); - tracing::debug!("batch_update_blocking done, updated files: {:?}", events.len()); - tracing::trace!("events: [{}]", events.to_short_form()); - events - } - - pub async fn delete_file(&self, path: PathBuf) -> Result<(), notify::Error> { - if !path.exists() { - return Err(notify::Error::new(notify::ErrorKind::Generic("File does not exist".to_string()))); - } - self.task.stop_watching_path(&path).await; - let result = self.task.handle_delete_update(path.clone()).await; - return result; - } - - pub fn delete_file_blocking(&self, path: PathBuf) -> Result<(), notify::Error> { - if !path.exists() { - return Err(notify::Error::new(notify::ErrorKind::Generic("File does not exist".to_string()))); - } - self.task.stop_watching_path_blocking(&path); - let result = self.task.handle_delete_update_blocking(path.clone()); - return result; - } - - pub fn has_events_pending(&self) -> bool { - self.output_rx.size_hint().0 > 0 - } - - pub fn try_next(&mut self) -> Option { - let res: Result, futures::channel::mpsc::TryRecvError> = self.output_rx.try_next(); - if res.is_err() { - return None; - } - res.unwrap() - } - - pub async fn next(&mut self) -> Option { - self.output_rx.next().await - } - - pub async fn next_timeout(&mut self, timeout: Duration) -> Option { - let res = tokio::time::timeout(timeout, self.output_rx.next()).await; - if res.is_err() { - return None; - } - res.unwrap() - } - - pub fn stop(&self) { - self.handle.abort(); - } - - pub fn get_all_files_blocking(&self) -> Vec<(PathBuf, FileContent)> { - let file_hashes = self.task.file_hashes.blocking_lock(); - file_hashes.iter().filter_map(|(path, _hash)| { - if path.is_file() { - let content = std::fs::read(path); - if content.is_ok() { - return Some((path.clone(), FileContent::from_buf(content.unwrap()))); - } - } - None - }).collect() - } + fn increase_ulimit() { + #[cfg(not(target_os = "windows"))] + { + let mut new_soft_limit = MAX_OPEN_FILES; + let mut new_hard_limit = MAX_OPEN_FILES; + let previous_result = getrlimit(Resource::NOFILE); + if let Err(e) = previous_result { + tracing::error!("failed to get ulimit {:?}", e); + } else if let Ok((soft_limit, hard_limit)) = previous_result { + tracing::debug!("soft ulimit {:?}", soft_limit); + tracing::debug!("hard ulimit {:?}", hard_limit); + if hard_limit > MAX_OPEN_FILES { + new_hard_limit = hard_limit; + } + if soft_limit > MAX_OPEN_FILES { + new_soft_limit = soft_limit; + } + } + + if let Err(e) = setrlimit(Resource::NOFILE, new_soft_limit, new_hard_limit) { + tracing::error!("failed to set ulimit {:?}", e); + } + let result = getrlimit(Resource::NOFILE); + if let Err(e) = result { + tracing::error!("failed to set ulimit {:?}", e); + } else if let Ok((soft_limit, hard_limit)) = result { + if soft_limit < MAX_OPEN_FILES || hard_limit < MAX_OPEN_FILES { + tracing::error!( + "failed to set ulimit; soft ulimit {:?}, hard ulimit {:?}", + soft_limit, + hard_limit + ); + } + } + } + } + + fn spawn_with_runtime( + watch_path: PathBuf, + ignore_globs: Vec, + rt: Option, + ) -> Self { + // if macos, increase ulimit to 100000000 + Self::increase_ulimit(); + let (output_tx, output_rx) = futures::channel::mpsc::unbounded(); + let (input_tx, mut input_rx) = futures::channel::mpsc::unbounded(); + // Spawn the file system watcher in a separate task + let notify_config = Config::default() + .with_follow_symlinks(false) + .with_ignore_globs(ignore_globs.clone()); + let ignore_globs: Vec = ignore_globs + .into_iter() + .filter_map(|glob_str| Pattern::new(&glob_str).ok()) + .collect(); + let (notify_tx, mut notify_rx) = futures::channel::mpsc::unbounded(); + + let debouncer_config = notify_debouncer_mini::Config::default() + .with_timeout(Duration::from_millis(DEBOUNCE_TIME)) + .with_batch_mode(true) + .with_notify_config(notify_config); + + let mut debouncer = new_debouncer_opt::<_, RecommendedWatcher>( + debouncer_config, + move |event: Result, notify::Error>| { + notify_tx.unbounded_send(event).unwrap(); + }, + ) + .unwrap(); + debouncer + .watcher() + .watch(&watch_path, RecursiveMode::Recursive) + .unwrap(); + + let rt_handle = if rt.is_some() { + rt.as_ref().unwrap().handle().clone() + } else { + tokio::runtime::Handle::current() + }; + let task: FileSystemTask = FileSystemTask { + watch_path: watch_path.clone(), + file_hashes: Arc::new(Mutex::new(HashMap::new())), + ignore_globs: ignore_globs, + watcher: Arc::new(Mutex::new(debouncer)), + paused: Arc::new(AtomicBool::new(false)), + found_ignored_paths: Arc::new(Mutex::new(HashSet::new())), + }; + + let mut this_task = task.clone(); + + let handle = rt_handle.spawn(async move { + this_task + .main_loop(&mut notify_rx, &mut input_rx, &output_tx) + .await; + }); + Self { + task, + output_rx, + input_tx, + handle, + rt: rt, + } + } + + pub fn spawn(watch_path: PathBuf, ignore_globs: Vec) -> Self { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .thread_name("FileSystemDriver: watcher thread") + .enable_all() + .build() + .unwrap(); + Self::spawn_with_runtime(watch_path, ignore_globs, Some(rt)) + } + + fn get_existing_parent_path(path: &PathBuf) -> Option { + let mut parent = Some(path.as_path()); + while parent.is_some() { + if parent.as_ref().unwrap().exists() { + break; + } + parent = parent.unwrap().parent(); + } + if parent.is_some() { + return Some(parent.unwrap().to_path_buf()); + } + None + } + + async fn pause_task(&self) { + self.input_tx + .unbounded_send(FileSystemUpdateEvent::Pause) + .ok(); + while !self.task.is_paused() { + sleep(Duration::from_millis(100)).await; + } + } + + fn pause_task_blocking(&self) { + self.input_tx + .unbounded_send(FileSystemUpdateEvent::Pause) + .ok(); + while !self.task.is_paused() { + std::thread::sleep(Duration::from_millis(100)); + } + } + + async fn resume_task(&self) { + self.input_tx + .unbounded_send(FileSystemUpdateEvent::Resume) + .ok(); + while self.task.is_paused() { + sleep(Duration::from_millis(100)).await; + } + } + + fn resume_task_blocking(&self) { + self.input_tx + .unbounded_send(FileSystemUpdateEvent::Resume) + .ok(); + while self.task.is_paused() { + std::thread::sleep(Duration::from_millis(100)); + } + } + + pub async fn batch_update(&self, updates: Vec) { + self.pause_task().await; + { + let mut file_hashes = self.task.file_hashes.lock().await; + for update in updates { + match update { + FileSystemUpdateEvent::FileSaved(path, content) => { + if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { + file_hashes.insert(path.clone(), hash_str); + } else { + tracing::error!("failed to write file {:?}", path); + } + } + FileSystemUpdateEvent::FileDeleted(path) => { + file_hashes.remove(&path); + } + FileSystemUpdateEvent::Pause => { + continue; + } + FileSystemUpdateEvent::Resume => { + continue; + } + } + } + } + self.resume_task().await; + } + + #[instrument(skip_all, level = tracing::Level::INFO)] + pub fn batch_update_blocking( + &self, + updates: Vec, + ) -> Vec { + tracing::debug!("# of updates: {:?}", updates.len()); + tracing::trace!("updates: [{}]", updates.to_short_form()); + self.pause_task_blocking(); + tracing::trace!("batch_update_blocking after pause"); + let mut events: Vec = Vec::new(); + { + let mut file_hashes = self.task.file_hashes.blocking_lock(); + for update in updates { + match update { + FileSystemUpdateEvent::FileSaved(path, content) => { + let new_hash_str = content.to_hash(); + let mut modified = false; + let mut created = false; + if let Some(old_hash) = file_hashes.get(&path) { + if old_hash != &new_hash_str { + modified = true; + } + } else { + created = true; + } + if modified || created { + if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { + if new_hash_str != hash_str { + tracing::error!( + "THIS SHOULD NOT HAPPEN: file {:?} previous calced hash {:?} != written hash {:?}", + path, + new_hash_str, + hash_str + ); + } + if modified { + tracing::trace!( + "file {:?} changed, hash {} -> {}", + path, + file_hashes.get(&path).unwrap(), + new_hash_str + ); + events + .push(FileSystemEvent::FileModified(path.clone(), content)); + } else { + tracing::trace!( + "file {:?} created, hash {}", + path, + new_hash_str + ); + events + .push(FileSystemEvent::FileCreated(path.clone(), content)); + } + file_hashes.insert(path, hash_str); + } else { + tracing::error!("failed to write file {:?}", path); + } + } else { + tracing::debug!( + "file {:?} already exists with same hash {:?}", + path, + new_hash_str + ); + } + } + FileSystemUpdateEvent::FileDeleted(path) => { + let _ = std::fs::remove_file(&path); + if file_hashes.remove(&path).is_some() { + events.push(FileSystemEvent::FileDeleted(path)); + } + } + _ => { + continue; + } + } + } + } + tracing::trace!("batch_update_blocking done, before resume"); + self.resume_task_blocking(); + tracing::debug!( + "batch_update_blocking done, updated files: {:?}", + events.len() + ); + tracing::trace!("events: [{}]", events.to_short_form()); + events + } + + pub fn has_events_pending(&self) -> bool { + self.output_rx.size_hint().0 > 0 + } + + pub fn try_next(&mut self) -> Option { + let res: Result, futures::channel::mpsc::TryRecvError> = + self.output_rx.try_next(); + if res.is_err() { + return None; + } + res.unwrap() + } + + pub async fn next(&mut self) -> Option { + self.output_rx.next().await + } + + pub async fn next_timeout(&mut self, timeout: Duration) -> Option { + let res = tokio::time::timeout(timeout, self.output_rx.next()).await; + if res.is_err() { + return None; + } + res.unwrap() + } + + pub fn stop(&self) { + self.handle.abort(); + } + + pub fn get_all_files_blocking(&self) -> Vec<(PathBuf, FileContent)> { + let file_hashes = self.task.file_hashes.blocking_lock(); + file_hashes + .iter() + .filter_map(|(path, _hash)| { + if path.is_file() { + let content = std::fs::read(path); + if content.is_ok() { + return Some((path.clone(), FileContent::from_buf(content.unwrap()))); + } + } + None + }) + .collect() + } } diff --git a/rust/src/fs/file_system_driver/tests.rs b/rust/src/fs/file_system_driver/tests.rs index 214329d6..d07bf6d0 100644 --- a/rust/src/fs/file_system_driver/tests.rs +++ b/rust/src/fs/file_system_driver/tests.rs @@ -1,421 +1,421 @@ -use std::collections::HashSet; -use std::fs::File; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::time::Duration; -use crate::fs::file_system_driver::{DEBOUNCE_TIME, FileSystemDriver, FileSystemEvent, FileSystemUpdateEvent}; -use crate::fs::file_utils::FileContent; -use tempfile::tempdir; -use tokio::{time::{sleep}}; -use crate::helpers::utils::ToShortForm; - -const WAIT_TIME: u64 = DEBOUNCE_TIME * 2; - -fn replace_res_prefix(watch_path: &PathBuf, path: &Path) -> PathBuf { - if path.to_string_lossy().starts_with("res://") { - return watch_path.join(path.to_string_lossy().replace("res://", "")); - } - path.to_path_buf() -} -// Helper function to normalize paths for comparison -fn normalize_path(watch_path: &PathBuf, path: &Path) -> PathBuf { - // On macOS, /var is a symlink to /private/var, so we need to resolve it - // if it begins with res://, replace it with the watch_path - let path = replace_res_prefix(watch_path, path); - if cfg!(target_os = "macos") { - // Try to canonicalize the path, which resolves symlinks - if let Ok(canonical) = path.canonicalize() { - return canonical; - } - } - // If canonicalization fails or we're not on macOS, just return the path as is - path -} - -#[tokio::test] -async fn test_file_system_watcher() { - // Create a temporary directory for testing - let dir = tempdir().unwrap(); - // if macos, add /private/ to the start of the path - let dir_path = dir.path().to_path_buf(); - - // Create the file system driver - let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec!["*.tmp".to_string()], None); - - - // Give the watcher time to initialize - sleep(Duration::from_millis(2000)).await; - - // Create a test file - let test_file = dir_path.join("test.txt"); - { - let mut file = File::create(&test_file).unwrap(); - file.write_all(b"test content").unwrap(); - file.sync_all().unwrap(); - } - - // Wait for the create event - if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { - match event { - FileSystemEvent::FileCreated(path, content) => { - assert_eq!(path, test_file); - assert_eq!(content, FileContent::String("test content".to_string())); - } - _ => panic!("Unexpected event"), - } - } - - let test_file2 = dir_path.join("test2.txt"); - { - let mut file = File::create(&test_file2).unwrap(); - file.write_all(b"test content").unwrap(); - file.sync_all().unwrap(); - } - - if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { - match event { - FileSystemEvent::FileCreated(path, content) => { - assert_eq!(path, test_file2); - assert_eq!(content, FileContent::String("test content".to_string())); - } - _ => panic!("Unexpected event"), - } - } - - // Modify the file with different content - { - let mut file = File::options().write(true).open(&test_file).unwrap(); - file.write_all(b"modified content").unwrap(); - file.sync_all().unwrap(); - } - - // Wait for the modify event - if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { - match event { - FileSystemEvent::FileModified(path, content) => { - assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); - assert_eq!(content, FileContent::String("modified content".to_string())); - } - _ => panic!("Unexpected event"), - } - } - - // Modify the file with the same content (should not trigger an event) - let mut file = File::options().write(true).open(&test_file).unwrap(); - file.write_all(b"modified content").unwrap(); - - // Delete the file - std::fs::remove_file(&test_file).unwrap(); - - // Wait for the delete event - // have it timeout after 100ms - if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { - match event { - FileSystemEvent::FileDeleted(path) => { - assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); - } - _ => panic!("Unexpected event"), - } - } else { - panic!("No event received"); - } -} - -#[tokio::test] -async fn test_ignore_globs() { - // Create a temporary directory for testing - let dir = tempdir().unwrap(); - // if macos, add /private/ to the start of the path - let dir_path = if cfg!(target_os = "macos") { - let mut path = dir.path().to_path_buf(); - let private = PathBuf::from("/private"); - path = private.join(path); - path - } else { - dir.path().to_path_buf() - }; - - // Create the file system driver with ignore globs - let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec!["*.tmp".to_string()], None); - - // Give the watcher time to initialize - sleep(Duration::from_millis(100)).await; - - - // Create a test file that should not be ignored - let test_file = dir_path.join("test.txt"); - let mut file = File::create(&test_file).unwrap(); - file.write_all(b"test content").unwrap(); - - - - // Wait for the create event (should only be for the non-ignored file) - if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { - match event { - FileSystemEvent::FileCreated(path, content) => { - assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); - assert_eq!(content, FileContent::String("test content".to_string())); - } - _ => panic!("Unexpected event"), - } - } else { - panic!("No event received"); - } - // Create a test file that should be ignored - let ignored_file = dir_path.join("test.tmp"); - let mut file = File::create(&ignored_file).unwrap(); - file.write_all(b"test content").unwrap(); - - - if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { - panic!("Unexpected event {:?}", event.to_short_form()); - } - { - // Modify the ignored file (should not trigger an event) - let mut file = File::options().write(true).open(&ignored_file).unwrap(); - file.write_all(b"modified content").unwrap(); - } - { - // Modify the non-ignored file (should trigger an event) - let mut file = File::options().write(true).open(&test_file).unwrap(); - file.write_all(b"modified content").unwrap(); - // close it - file.sync_all().unwrap(); - } - // Wait for the modify event (should only be for the non-ignored file) - if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { - match event { - FileSystemEvent::FileModified(path, _) => { - assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); - } - _ => panic!("Unexpected event"), - } - } else { - panic!("No event received"); - } - - // Delete both files - std::fs::remove_file(&test_file).unwrap(); - - // Wait for the delete event (should only be for the non-ignored file) - if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { - match event { - FileSystemEvent::FileDeleted(path) => { - assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); - } - _ => panic!("Unexpected event"), - } - } else { - panic!("No event received"); - } - std::fs::remove_file(&ignored_file).unwrap(); - - // try_next should return None - assert!(driver.next_timeout(Duration::from_millis(WAIT_TIME)).await.is_none()); - -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 1)] -async fn test_file_system_update_events() { - // Create a temporary directory for testing - let dir = tempdir().unwrap(); - // if macos, add /private/ to the start of the path - let dir_path = dir.path().to_path_buf(); - - // Create the file system driver - let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec![], None); - - // Give the watcher time to initialize - sleep(Duration::from_millis(100)).await; - let test_content = "test content"; - let modified_content = "modified content"; - // Create a file via update event - let test_file = dir_path.join("test.txt"); - let result = driver.save_file(test_file.clone(), FileContent::String(test_content.to_string())).await; - assert!(result.is_ok()); - // sleep - sleep(Duration::from_millis(WAIT_TIME)).await; - // check that there's no event for the file created - assert!(driver.try_next().is_none()); - - // check that the file exists and contains the test_content - assert!(test_file.exists()); - assert_eq!(std::fs::read_to_string(&test_file).unwrap(), test_content); - // modify the file - let result = driver.save_file(test_file.clone(), FileContent::String(modified_content.to_string())).await; - assert!(result.is_ok()); - sleep(Duration::from_millis(WAIT_TIME)).await; - - // check that there's no event for the file modified - assert!(driver.try_next().is_none()); - // check that the file exists and contains the modified_content - assert!(test_file.exists()); - assert_eq!(std::fs::read_to_string(&test_file).unwrap(), modified_content); - // delete the file - let result = driver.delete_file(test_file.clone()).await; - assert!(result.is_ok()); - sleep(Duration::from_millis(WAIT_TIME)).await; - // check that there's no event for the file modified - assert!(driver.try_next().is_none()); - - // check that the file does not exist - assert!(!test_file.exists()); -} - - -#[tokio::test(flavor = "multi_thread", worker_threads = 1)] -async fn test_file_system_large_number_of_files() { - // we want to have at least 1000 files in the directory to test that we actually do raise the ulimit - let dir = tempdir().unwrap(); - let dir_path = dir.path().to_path_buf(); - let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec!["*.tmp".to_string()], None); - // give the watcher time to initialize - sleep(Duration::from_millis(100)).await; - // create 1000 files - let mut test_paths = HashSet::new(); - for i in 0..1000 { - let test_path = dir_path.join(format!("test_{}.txt", i)); - let mut file = File::create(&test_path).unwrap(); - file.write_all(b"test content").unwrap(); - file.sync_all().unwrap(); - test_paths.insert(test_path); - } - // wait for the watcher to process the events - sleep(Duration::from_millis(100)).await; - // check that the files exist - // for i in 0..1000 { - // let file = dir_path.join(format!("test_{}.txt", i)); - // assert!(file.exists()); - // } - // now check to see if we have 1000 events - let mut found_paths = HashSet::new(); - while let Some(event) = driver.next_timeout(Duration::from_millis(10000)).await { - let event_path = if let FileSystemEvent::FileCreated(path, _) = event { - path - } else if let FileSystemEvent::FileModified(path, _) = event { - path - } else { - panic!("Unexpected event type {:?}", event.to_short_form()); - }; - found_paths.insert(event_path); - if found_paths.len() == test_paths.len() { - break; - } - } - // assert_eq!(count, 1000); - for path in test_paths { - assert!(found_paths.contains(&path)); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 1)] -async fn test_file_system_batch_update() { - let dir = tempdir().unwrap(); - let dir_path = dir.path().to_path_buf(); - let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec![], None); - // update 1000 files - let mut updates = Vec::new(); - for i in 0..1000 { - let test_path = dir_path.join(format!("test_{}.txt", i)); - updates.push(FileSystemUpdateEvent::FileSaved(test_path, FileContent::String("test content".to_string()))); - } - driver.batch_update(updates).await; - // wait for the watcher to process the events - sleep(Duration::from_millis(100)).await; - // check that the files exist - { - let hashes = driver.task.file_hashes.clone(); - let hash_table = hashes.lock().await; - for i in 0..1000 { - let file = dir_path.join(format!("test_{}.txt", i)); - assert!(file.exists()); - assert!(hash_table.contains_key(&file)); - } - } - // there should be no events emitted - if let Some(event) = driver.next_timeout(Duration::from_millis(100)).await { - assert!(false, "Unexpected event type {:?}", event.to_short_form()); - } - sleep(Duration::from_millis(1000)).await; - // write a single file - let test_path = dir_path.join("test_woop.txt"); - let mut file = File::create(&test_path).unwrap(); - file.write_all(b"test content").unwrap(); - file.sync_all().unwrap(); - sleep(Duration::from_millis(100)).await; - // check that the file exists - assert!(test_path.exists()); - // check that we got an event for it - if let Some(event) = driver.next().await { - match event { - FileSystemEvent::FileCreated(path, content) => { - assert_eq!(path, test_path); - assert_eq!(content, FileContent::String("test content".to_string())); - } - _ => panic!("Unexpected event"), - } - } else { - panic!("No event received"); - } - -} -// test blocking file system update -#[test] -fn test_file_system_blocking_update() { - // set the default tokio runtime to multi_thread - let dir = tempdir().unwrap(); - let dir_path = dir.path().to_path_buf(); - // spawn with the default runtime - let mut driver = FileSystemDriver::spawn(dir_path.clone(), vec![]); - // update 1000 files - let mut updates = Vec::new(); - for i in 0..1000 { - let test_path = dir_path.join(format!("test_{}.txt", i)); - updates.push(FileSystemUpdateEvent::FileSaved(test_path, FileContent::String("test content".to_string()))); - } - driver.batch_update_blocking(updates); - // wait for the watcher to process the events - // check that the files exist - { - let hashes = driver.task.file_hashes.clone(); - let hash_table = hashes.blocking_lock(); - for i in 0..1000 { - let file = dir_path.join(format!("test_{}.txt", i)); - assert!(file.exists()); - assert!(hash_table.contains_key(&file)); - } - } - // there should be no events emitted - if let Some(event) = driver.try_next() { - assert!(false, "Unexpected event type {:?}", event.to_short_form()); - } - std::thread::sleep(Duration::from_millis(1000)); - // write a single file - let test_path = dir_path.join("test_woop.txt"); - let mut file = File::create(&test_path).unwrap(); - file.write_all(b"test content").unwrap(); - file.sync_all().unwrap(); - std::thread::sleep(Duration::from_millis(1000)); - // check that the file exists - assert!(test_path.exists()); - // check that we got an event for it - if let Some(event) = driver.try_next() { - match event { - FileSystemEvent::FileCreated(path, content) => { - assert_eq!(path, test_path); - assert_eq!(content, FileContent::String("test content".to_string())); - } - _ => panic!("Unexpected event"), - } - } else { - panic!("No event received"); - } -} +// use std::collections::HashSet; +// use std::fs::File; +// use std::io::Write; +// use std::path::{Path, PathBuf}; +// use std::time::Duration; +// use crate::fs::file_system_driver::{DEBOUNCE_TIME, FileSystemDriver, FileSystemEvent, FileSystemUpdateEvent}; +// use crate::fs::file_utils::FileContent; +// use tempfile::tempdir; +// use tokio::{time::{sleep}}; +// use crate::helpers::utils::ToShortForm; + +// const WAIT_TIME: u64 = DEBOUNCE_TIME * 2; + +// fn replace_res_prefix(watch_path: &PathBuf, path: &Path) -> PathBuf { +// if path.to_string_lossy().starts_with("res://") { +// return watch_path.join(path.to_string_lossy().replace("res://", "")); +// } +// path.to_path_buf() +// } +// // Helper function to normalize paths for comparison +// fn normalize_path(watch_path: &PathBuf, path: &Path) -> PathBuf { +// // On macOS, /var is a symlink to /private/var, so we need to resolve it +// // if it begins with res://, replace it with the watch_path +// let path = replace_res_prefix(watch_path, path); +// if cfg!(target_os = "macos") { +// // Try to canonicalize the path, which resolves symlinks +// if let Ok(canonical) = path.canonicalize() { +// return canonical; +// } +// } +// // If canonicalization fails or we're not on macOS, just return the path as is +// path +// } + +// #[tokio::test] +// async fn test_file_system_watcher() { +// // Create a temporary directory for testing +// let dir = tempdir().unwrap(); +// // if macos, add /private/ to the start of the path +// let dir_path = dir.path().to_path_buf(); + +// // Create the file system driver +// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec!["*.tmp".to_string()], None); + + +// // Give the watcher time to initialize +// sleep(Duration::from_millis(2000)).await; + +// // Create a test file +// let test_file = dir_path.join("test.txt"); +// { +// let mut file = File::create(&test_file).unwrap(); +// file.write_all(b"test content").unwrap(); +// file.sync_all().unwrap(); +// } + +// // Wait for the create event +// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { +// match event { +// FileSystemEvent::FileCreated(path, content) => { +// assert_eq!(path, test_file); +// assert_eq!(content, FileContent::String("test content".to_string())); +// } +// _ => panic!("Unexpected event"), +// } +// } + +// let test_file2 = dir_path.join("test2.txt"); +// { +// let mut file = File::create(&test_file2).unwrap(); +// file.write_all(b"test content").unwrap(); +// file.sync_all().unwrap(); +// } + +// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { +// match event { +// FileSystemEvent::FileCreated(path, content) => { +// assert_eq!(path, test_file2); +// assert_eq!(content, FileContent::String("test content".to_string())); +// } +// _ => panic!("Unexpected event"), +// } +// } + +// // Modify the file with different content +// { +// let mut file = File::options().write(true).open(&test_file).unwrap(); +// file.write_all(b"modified content").unwrap(); +// file.sync_all().unwrap(); +// } + +// // Wait for the modify event +// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { +// match event { +// FileSystemEvent::FileModified(path, content) => { +// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); +// assert_eq!(content, FileContent::String("modified content".to_string())); +// } +// _ => panic!("Unexpected event"), +// } +// } + +// // Modify the file with the same content (should not trigger an event) +// let mut file = File::options().write(true).open(&test_file).unwrap(); +// file.write_all(b"modified content").unwrap(); + +// // Delete the file +// std::fs::remove_file(&test_file).unwrap(); + +// // Wait for the delete event +// // have it timeout after 100ms +// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { +// match event { +// FileSystemEvent::FileDeleted(path) => { +// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); +// } +// _ => panic!("Unexpected event"), +// } +// } else { +// panic!("No event received"); +// } +// } + +// #[tokio::test] +// async fn test_ignore_globs() { +// // Create a temporary directory for testing +// let dir = tempdir().unwrap(); +// // if macos, add /private/ to the start of the path +// let dir_path = if cfg!(target_os = "macos") { +// let mut path = dir.path().to_path_buf(); +// let private = PathBuf::from("/private"); +// path = private.join(path); +// path +// } else { +// dir.path().to_path_buf() +// }; + +// // Create the file system driver with ignore globs +// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec!["*.tmp".to_string()], None); + +// // Give the watcher time to initialize +// sleep(Duration::from_millis(100)).await; + + +// // Create a test file that should not be ignored +// let test_file = dir_path.join("test.txt"); +// let mut file = File::create(&test_file).unwrap(); +// file.write_all(b"test content").unwrap(); + + + +// // Wait for the create event (should only be for the non-ignored file) +// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { +// match event { +// FileSystemEvent::FileCreated(path, content) => { +// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); +// assert_eq!(content, FileContent::String("test content".to_string())); +// } +// _ => panic!("Unexpected event"), +// } +// } else { +// panic!("No event received"); +// } +// // Create a test file that should be ignored +// let ignored_file = dir_path.join("test.tmp"); +// let mut file = File::create(&ignored_file).unwrap(); +// file.write_all(b"test content").unwrap(); + + +// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { +// panic!("Unexpected event {:?}", event.to_short_form()); +// } +// { +// // Modify the ignored file (should not trigger an event) +// let mut file = File::options().write(true).open(&ignored_file).unwrap(); +// file.write_all(b"modified content").unwrap(); +// } +// { +// // Modify the non-ignored file (should trigger an event) +// let mut file = File::options().write(true).open(&test_file).unwrap(); +// file.write_all(b"modified content").unwrap(); +// // close it +// file.sync_all().unwrap(); +// } +// // Wait for the modify event (should only be for the non-ignored file) +// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { +// match event { +// FileSystemEvent::FileModified(path, _) => { +// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); +// } +// _ => panic!("Unexpected event"), +// } +// } else { +// panic!("No event received"); +// } + +// // Delete both files +// std::fs::remove_file(&test_file).unwrap(); + +// // Wait for the delete event (should only be for the non-ignored file) +// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { +// match event { +// FileSystemEvent::FileDeleted(path) => { +// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); +// } +// _ => panic!("Unexpected event"), +// } +// } else { +// panic!("No event received"); +// } +// std::fs::remove_file(&ignored_file).unwrap(); + +// // try_next should return None +// assert!(driver.next_timeout(Duration::from_millis(WAIT_TIME)).await.is_none()); + +// } // #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -// // run the test_file_system_large_number_of_files test 100 times -// async fn test_file_system_large_number_of_files_100() { -// for _ in 0..100 { -// test_file_system_large_number_of_files().await; -// } +// async fn test_file_system_update_events() { +// // Create a temporary directory for testing +// let dir = tempdir().unwrap(); +// // if macos, add /private/ to the start of the path +// let dir_path = dir.path().to_path_buf(); + +// // Create the file system driver +// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec![], None); + +// // Give the watcher time to initialize +// sleep(Duration::from_millis(100)).await; +// let test_content = "test content"; +// let modified_content = "modified content"; +// // Create a file via update event +// let test_file = dir_path.join("test.txt"); +// let result = driver.save_file(test_file.clone(), FileContent::String(test_content.to_string())).await; +// assert!(result.is_ok()); +// // sleep +// sleep(Duration::from_millis(WAIT_TIME)).await; +// // check that there's no event for the file created +// assert!(driver.try_next().is_none()); + +// // check that the file exists and contains the test_content +// assert!(test_file.exists()); +// assert_eq!(std::fs::read_to_string(&test_file).unwrap(), test_content); +// // modify the file +// let result = driver.save_file(test_file.clone(), FileContent::String(modified_content.to_string())).await; +// assert!(result.is_ok()); +// sleep(Duration::from_millis(WAIT_TIME)).await; + +// // check that there's no event for the file modified +// assert!(driver.try_next().is_none()); +// // check that the file exists and contains the modified_content +// assert!(test_file.exists()); +// assert_eq!(std::fs::read_to_string(&test_file).unwrap(), modified_content); +// // delete the file +// let result = driver.delete_file(test_file.clone()).await; +// assert!(result.is_ok()); +// sleep(Duration::from_millis(WAIT_TIME)).await; +// // check that there's no event for the file modified +// assert!(driver.try_next().is_none()); + +// // check that the file does not exist +// assert!(!test_file.exists()); // } + + +// #[tokio::test(flavor = "multi_thread", worker_threads = 1)] +// async fn test_file_system_large_number_of_files() { +// // we want to have at least 1000 files in the directory to test that we actually do raise the ulimit +// let dir = tempdir().unwrap(); +// let dir_path = dir.path().to_path_buf(); +// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec!["*.tmp".to_string()], None); +// // give the watcher time to initialize +// sleep(Duration::from_millis(100)).await; +// // create 1000 files +// let mut test_paths = HashSet::new(); +// for i in 0..1000 { +// let test_path = dir_path.join(format!("test_{}.txt", i)); +// let mut file = File::create(&test_path).unwrap(); +// file.write_all(b"test content").unwrap(); +// file.sync_all().unwrap(); +// test_paths.insert(test_path); +// } +// // wait for the watcher to process the events +// sleep(Duration::from_millis(100)).await; +// // check that the files exist +// // for i in 0..1000 { +// // let file = dir_path.join(format!("test_{}.txt", i)); +// // assert!(file.exists()); +// // } +// // now check to see if we have 1000 events +// let mut found_paths = HashSet::new(); +// while let Some(event) = driver.next_timeout(Duration::from_millis(10000)).await { +// let event_path = if let FileSystemEvent::FileCreated(path, _) = event { +// path +// } else if let FileSystemEvent::FileModified(path, _) = event { +// path +// } else { +// panic!("Unexpected event type {:?}", event.to_short_form()); +// }; +// found_paths.insert(event_path); +// if found_paths.len() == test_paths.len() { +// break; +// } +// } +// // assert_eq!(count, 1000); +// for path in test_paths { +// assert!(found_paths.contains(&path)); +// } +// } + +// #[tokio::test(flavor = "multi_thread", worker_threads = 1)] +// async fn test_file_system_batch_update() { +// let dir = tempdir().unwrap(); +// let dir_path = dir.path().to_path_buf(); +// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec![], None); +// // update 1000 files +// let mut updates = Vec::new(); +// for i in 0..1000 { +// let test_path = dir_path.join(format!("test_{}.txt", i)); +// updates.push(FileSystemUpdateEvent::FileSaved(test_path, FileContent::String("test content".to_string()))); +// } +// driver.batch_update(updates).await; +// // wait for the watcher to process the events +// sleep(Duration::from_millis(100)).await; +// // check that the files exist +// { +// let hashes = driver.task.file_hashes.clone(); +// let hash_table = hashes.lock().await; +// for i in 0..1000 { +// let file = dir_path.join(format!("test_{}.txt", i)); +// assert!(file.exists()); +// assert!(hash_table.contains_key(&file)); +// } +// } +// // there should be no events emitted +// if let Some(event) = driver.next_timeout(Duration::from_millis(100)).await { +// assert!(false, "Unexpected event type {:?}", event.to_short_form()); +// } +// sleep(Duration::from_millis(1000)).await; +// // write a single file +// let test_path = dir_path.join("test_woop.txt"); +// let mut file = File::create(&test_path).unwrap(); +// file.write_all(b"test content").unwrap(); +// file.sync_all().unwrap(); +// sleep(Duration::from_millis(100)).await; +// // check that the file exists +// assert!(test_path.exists()); +// // check that we got an event for it +// if let Some(event) = driver.next().await { +// match event { +// FileSystemEvent::FileCreated(path, content) => { +// assert_eq!(path, test_path); +// assert_eq!(content, FileContent::String("test content".to_string())); +// } +// _ => panic!("Unexpected event"), +// } +// } else { +// panic!("No event received"); +// } + +// } +// // test blocking file system update +// #[test] +// fn test_file_system_blocking_update() { +// // set the default tokio runtime to multi_thread +// let dir = tempdir().unwrap(); +// let dir_path = dir.path().to_path_buf(); +// // spawn with the default runtime +// let mut driver = FileSystemDriver::spawn(dir_path.clone(), vec![]); +// // update 1000 files +// let mut updates = Vec::new(); +// for i in 0..1000 { +// let test_path = dir_path.join(format!("test_{}.txt", i)); +// updates.push(FileSystemUpdateEvent::FileSaved(test_path, FileContent::String("test content".to_string()))); +// } +// driver.batch_update_blocking(updates); +// // wait for the watcher to process the events +// // check that the files exist +// { +// let hashes = driver.task.file_hashes.clone(); +// let hash_table = hashes.blocking_lock(); +// for i in 0..1000 { +// let file = dir_path.join(format!("test_{}.txt", i)); +// assert!(file.exists()); +// assert!(hash_table.contains_key(&file)); +// } +// } +// // there should be no events emitted +// if let Some(event) = driver.try_next() { +// assert!(false, "Unexpected event type {:?}", event.to_short_form()); +// } +// std::thread::sleep(Duration::from_millis(1000)); +// // write a single file +// let test_path = dir_path.join("test_woop.txt"); +// let mut file = File::create(&test_path).unwrap(); +// file.write_all(b"test content").unwrap(); +// file.sync_all().unwrap(); +// std::thread::sleep(Duration::from_millis(1000)); +// // check that the file exists +// assert!(test_path.exists()); +// // check that we got an event for it +// if let Some(event) = driver.try_next() { +// match event { +// FileSystemEvent::FileCreated(path, content) => { +// assert_eq!(path, test_path); +// assert_eq!(content, FileContent::String("test content".to_string())); +// } +// _ => panic!("Unexpected event"), +// } +// } else { +// panic!("No event received"); +// } +// } + +// // #[tokio::test(flavor = "multi_thread", worker_threads = 1)] +// // // run the test_file_system_large_number_of_files test 100 times +// // async fn test_file_system_large_number_of_files_100() { +// // for _ in 0..100 { +// // test_file_system_large_number_of_files().await; +// // } +// // } diff --git a/rust/src/fs/file_utils.rs b/rust/src/fs/file_utils.rs index ee4b5044..b7540f2b 100644 --- a/rust/src/fs/file_utils.rs +++ b/rust/src/fs/file_utils.rs @@ -5,8 +5,8 @@ use std::path::{PathBuf}; use std::str; use automerge::{Automerge, ChangeHash, ObjType, ReadDoc}; use automerge::ObjId; +use md5::Digest; use samod::{DocumentId}; -use ya_md5::{Md5Hasher}; use crate::helpers::doc_utils::SimpleDocReader; use crate::helpers::utils::{ToShortForm, parse_automerge_url}; @@ -20,11 +20,18 @@ pub enum FileContent { Deleted, } +#[derive(Debug)] +pub enum FileSystemEvent { + FileCreated(PathBuf, FileContent), + FileModified(PathBuf, FileContent), + FileDeleted(PathBuf), +} + impl FileContent { // Write file content to disk - pub fn write_file_content(path: &PathBuf, content: &FileContent) -> std::io::Result { + async fn write_file_content(path: &PathBuf, content: &FileContent) -> std::io::Result { // Check if the file exists - let mut _temp_text: Option = None; + let mut temp_text: Option = None; // Write the content based on its type let buf: &[u8] = match content { FileContent::String(text) => { @@ -34,18 +41,18 @@ impl FileContent { data } FileContent::Scene(scene) => { - _temp_text = Some(scene.serialize()); - _temp_text.as_ref().unwrap().as_bytes() + temp_text = Some(scene.serialize()); + temp_text.as_ref().unwrap().as_bytes() } FileContent::Deleted => { return Err(std::io::Error::new(std::io::ErrorKind::Other, "Failed to write file")); } }; - let hash = Md5Hasher::hash_slice(buf).to_string(); + let hash = md5::compute(buf); // ensure the directory exists if let Some(dir) = path.parent() { if !dir.exists() { - std::fs::create_dir_all(dir)?; + tokio::fs::create_dir_all(dir).await?; } } // Open the file with the appropriate mode @@ -63,8 +70,8 @@ impl FileContent { Ok(hash) } - pub fn write(&self, path: &PathBuf) -> std::io::Result { - FileContent::write_file_content(path, self) + pub async fn write(&self, path: &PathBuf) -> std::io::Result { + FileContent::write_file_content(path, self).await } // pub fn from_path(path: &PathBuf) -> Option { @@ -102,12 +109,12 @@ impl FileContent { FileContent::from_string(string) } - pub fn to_hash(&self) -> String { + pub fn to_hash(&self) -> Digest { match self { - FileContent::String(s) => Md5Hasher::hash_slice(s.as_bytes()).to_string(), - FileContent::Binary(bytes) => Md5Hasher::hash_slice(bytes.as_slice()).to_string(), - FileContent::Scene(scene) => Md5Hasher::hash_slice(scene.serialize().as_bytes()).to_string(), - FileContent::Deleted => "".to_string(), + FileContent::String(s) => md5::compute(s.as_bytes()), + FileContent::Binary(bytes) => md5::compute(bytes.as_slice()), + FileContent::Scene(scene) => md5::compute(scene.serialize().as_bytes()), + FileContent::Deleted => md5::compute(""), } } @@ -188,35 +195,31 @@ impl Default for &FileContent { } } -pub fn calculate_file_hash(path: &PathBuf) -> Option { +pub async fn calculate_file_hash(path: &PathBuf) -> Option { if !path.is_file() { return None; } - let mut file = match File::open(path) { + let mut file = match tokio::fs::read(path).await { Ok(file) => file, Err(_) => return None, }; - match Md5Hasher::hash(&mut file) { - Ok(hash) => Some(format!("{}", hash)), - Err(_) => None, - } + return Some(md5::compute(&mut file)); } // get the buffer and hash of a file -pub fn get_buffer_and_hash(path: &PathBuf) -> Result<(Vec, String), io::Error> { +pub async fn get_buffer_and_hash(path: &PathBuf) -> Result<(Vec, Digest), tokio::io::Error> { if !path.is_file() { return Err(io::Error::new(io::ErrorKind::Other, "Not a file")); } - let buf = std::fs::read(path); + let buf = tokio::fs::read(path).await; if buf.is_err() { return Err(io::Error::new(io::ErrorKind::Other, "Failed to read file")); } let buf = buf.unwrap(); - let hash = Md5Hasher::hash_slice(&buf); - let hash_str = format!("{}", hash); - Ok((buf, hash_str)) + let hash = md5::compute(&buf); + Ok((buf, hash)) } pub fn is_buf_binary(buf: &[u8]) -> bool { diff --git a/rust/src/helpers/utils.rs b/rust/src/helpers/utils.rs index 3e0f62c7..f8e86198 100644 --- a/rust/src/helpers/utils.rs +++ b/rust/src/helpers/utils.rs @@ -2,7 +2,7 @@ use std::{ collections::{HashMap, HashSet}, fmt, path::Path, str::FromStr, time::{SystemTime, UNIX_EPOCH} }; -use crate::{diff::differ::ProjectDiff, helpers::{branch::BranchState, doc_utils::SimpleDocReader}}; +use crate::{diff::differ::ProjectDiff, helpers::{branch::BranchState, doc_utils::SimpleDocReader}, project::branch_db::HistoryRef}; use automerge::{ Automerge, Change, ChangeHash, Patch, PatchLog, ROOT, ReadDoc, transaction::{CommitOptions, Transaction} }; @@ -10,19 +10,6 @@ use samod::{DocHandle, DocumentId}; use chrono::{DateTime, Local}; use serde::{Deserialize, Serialize}; -// These functions are for compatibilities sake, and they will be removed in the future -#[inline(always)] -pub(crate) fn get_default_patch_log() -> PatchLog { - #[cfg(not(feature = "automerge_0_6"))] - { - PatchLog::inactive() - } - #[cfg(feature = "automerge_0_6")] - { - PatchLog::inactive(automerge::patches::TextRepresentation::String(automerge::TextEncoding::Utf8CodeUnit)) - } -} - #[inline(always)] pub(crate) fn get_automerge_doc_diff(doc: &Automerge, old_heads: &[ChangeHash], new_heads: &[ChangeHash]) -> Vec { #[cfg(not(feature = "automerge_0_6"))] @@ -36,7 +23,7 @@ pub(crate) fn get_automerge_doc_diff(doc: &Automerge, old_heads: &[ChangeHash], } -pub(crate) fn get_changed_files_vec(patches: &Vec) -> Vec { +pub(crate) fn get_changed_files(patches: &Vec) -> HashSet { let mut changed_files = HashSet::new(); // log all patches @@ -65,47 +52,7 @@ pub(crate) fn get_changed_files_vec(patches: &Vec) -> Vec>(); -} - - -pub(crate) fn get_linked_docs_of_branch( - branch_doc_handle: &DocHandle, -) -> HashMap { - // Collect all linked doc IDs from this branch - branch_doc_handle.with_document(|d| { - let files = match d.get_obj_id(ROOT, "files") { - Some(files) => files, - None => { - tracing::warn!( - "Failed to load files for branch doc {:?}", - branch_doc_handle.document_id() - ); - return HashMap::new(); - } - }; - - d.keys(&files) - .filter_map(|path| { - let file = match d.get_obj_id(&files, &path) { - Some(file) => file, - None => { - tracing::error!("Failed to load linked doc {:?}", path); - return None; - } - }; - - let url = match d.get_string(&file, "url") { - Some(url) => url, - None => { - return None; - } - }; - - parse_automerge_url(&url).map(|id| (path.clone(), id)) - }) - .collect::>() - }) + return changed_files; } pub(crate) fn parse_automerge_url(url: &str) -> Option { @@ -195,27 +142,6 @@ pub struct CommitInfo { pub summary: String } -impl From<&&Change> for CommitInfo { - fn from(change: &&Change) -> Self { - CommitInfo::from(*change) - } -} - -impl From<&Change> for CommitInfo { - fn from(change: &Change) -> Self { - CommitInfo { - hash: change.hash(), - timestamp: change.timestamp(), - metadata: change.message().and_then(|m| serde_json::from_str::(&m).ok()), - - // set during ingestion - synced: false, - summary: "".to_string() - } - } -} - - #[derive(Debug)] pub struct BranchWrapper { pub state: BranchState, diff --git a/rust/src/interop/godot_project.rs b/rust/src/interop/godot_project.rs index d36d1a37..e00b9a6d 100644 --- a/rust/src/interop/godot_project.rs +++ b/rust/src/interop/godot_project.rs @@ -1,7 +1,8 @@ -use crate::fs::file_utils::FileContent; +use crate::fs::file_utils::{FileContent, FileSystemEvent}; use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkConfigAccessor, PatchworkEditorAccessor}; +use crate::project::driver::Driver; +use crate::project::new_project::{GodotProjectSignal, Project}; use crate::project::project_api::{BranchViewModel, ProjectViewModel}; -use crate::project::project::{Project, GodotProjectSignal}; use automerge::ChangeHash; use godot::classes::editor_plugin::DockSlot; use ::safer_ffi::prelude::*; @@ -14,13 +15,11 @@ use godot::classes::ResourceLoader; use godot::classes::{EditorPlugin, Engine, IEditorPlugin}; use godot::classes::{DirAccess}; use godot::prelude::*; -use godot::prelude::Dictionary; use tracing::instrument; use std::collections::{HashSet}; use std::path::PathBuf; use std::{collections::HashMap, str::FromStr}; use crate::interop::godot_helpers::{ToGodotExt, branch_view_model_to_dict, change_view_model_to_dict, diff_view_model_to_dict}; -use crate::fs::file_system_driver::{FileSystemEvent}; // This is the worst thing I've ever done // Get the file system @@ -204,7 +203,7 @@ impl GodotProject { } #[func] - fn get_sync_status(&self) -> Dictionary { + fn get_sync_status(&self) -> VarDictionary { self.project.get_sync_status().to_godot() } @@ -350,18 +349,18 @@ impl GodotProject { let mut file_created = false; let (abs_path, content) = match event { FileSystemEvent::FileCreated(path, content) => { - pending_editor_update.added_files.insert(self.project.localize_path(&path.to_string_lossy().to_string())); + pending_editor_update.added_files.insert(ProjectSettings::singleton().localize_path(&path.to_string_lossy().to_string()).to_string()); file_created = true; (path, content) }, FileSystemEvent::FileModified(path, content) => (path, content), FileSystemEvent::FileDeleted(path) => { - pending_editor_update.deleted_files.insert(self.project.localize_path(&path.to_string_lossy().to_string())); + pending_editor_update.deleted_files.insert(ProjectSettings::singleton().localize_path(&path.to_string_lossy().to_string()).to_string()); continue; }, }; files_changed.push(abs_path.to_string_lossy().to_string()); - let res_path = self.project.localize_path(&abs_path.to_string_lossy().to_string()); + let res_path = ProjectSettings::singleton().localize_path(&abs_path.to_string_lossy().to_string()).to_string(); let extension = abs_path.extension().unwrap_or_default().to_string_lossy().to_string().to_ascii_lowercase(); if extension == "gd" { pending_editor_update.scripts_to_reload.insert(res_path); @@ -415,7 +414,7 @@ impl GodotProject { if !self.pending_editor_update.any_changes() { return false; } - if !Project::safe_to_update_godot(false) { + if !Driver::safe_to_update_godot() { return false; } self.base_mut().set_process(false); @@ -441,7 +440,7 @@ impl INode for GodotProject { fn init(_base: Base) -> Self { GodotProject { base: _base, - project: Project::new(ProjectSettings::singleton().globalize_path("res://").to_string()), + project: Project::new(ProjectSettings::singleton().globalize_path("res://").to_string().into()), pending_editor_update: PendingEditorUpdate::default(), reload_project_settings_callable: None } @@ -464,7 +463,7 @@ impl INode for GodotProject { } fn exit_tree(&mut self) { - if self.project.is_started() { + if self.project.has_project() { self.project.stop(); } // Perform typical plugin operations here. @@ -472,7 +471,7 @@ impl INode for GodotProject { #[instrument(target = "patchwork_rust_core::godot_project::outer_process", level = tracing::Level::DEBUG, skip_all)] fn process(&mut self, _delta: f64) { - if !self.project.is_started() { + if !self.project.has_project() { return; } let (updates, signals) = self.project.process(_delta); diff --git a/rust/src/project.rs b/rust/src/project.rs index 0df55079..bbb50727 100644 --- a/rust/src/project.rs +++ b/rust/src/project.rs @@ -1,4 +1,15 @@ pub mod project_api; -mod project_driver; -pub mod project; +//mod project_driver; +//pub mod project; pub mod project_api_impl; +mod connection; +mod document_watcher; +mod fs_watcher; +mod sync_fs_to_automerge; +mod sync_automerge_to_fs; +// pub for use in differ; consider restructuring +pub mod branch_db; +mod peer_watcher; +pub mod new_project; +// TODO (Lilith): Make this not pub +pub mod driver; \ No newline at end of file diff --git a/rust/src/project/branch_db.rs b/rust/src/project/branch_db.rs new file mode 100644 index 00000000..e9f83f01 --- /dev/null +++ b/rust/src/project/branch_db.rs @@ -0,0 +1,168 @@ +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + sync::Arc, +}; + +use automerge::{Automerge, ChangeHash, ObjId, ObjType, ROOT, ReadDoc}; +use autosurgeon::Doc; +use samod::{DocHandle, DocumentId, Repo}; +use serde::{Deserialize, Serialize}; +use tokio::sync::{Mutex, RwLock}; + +use crate::{ + fs::file_utils::FileContent, + helpers::{ + branch::{BinaryDocState, BranchState, BranchesMetadataDoc}, + utils::{CommitMetadata, commit_with_attribution_and_timestamp}, + }, +}; + +mod branch; +mod commit; +mod file; +mod util; + +// TODO (Lilith): Move this to utils +/// Represents a location anywhere in Patchwork's history. +/// Associates a branch with heads on that branch. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HistoryRef { + /// The branch the ref is on. + pub branch: DocumentId, + // todo: it would be very nice to have a Heads struct + /// The Automerge heads for the history location + pub heads: Vec, +} + +impl HistoryRef { + pub fn is_valid(&self) -> bool { + return !self.heads.is_empty(); + } +} + +impl Eq for HistoryRef {} + +impl std::hash::Hash for HistoryRef { + fn hash(&self, state: &mut H) { + self.branch.hash(state); + self.heads.hash(state); + } +} + +impl PartialEq for HistoryRef { + fn eq(&self, other: &Self) -> bool { + self.branch == other.branch && self.heads == other.heads + } +} + +/// [BranchDb] is the primary data source for project data. +/// It stores the project state, and provides a handful of convenient state-manipulation methods for controllers to use. +#[derive(Clone, Debug)] +pub struct BranchDb { + // TODO (Lilith): consider swapping to RwLock + inner: Arc>, + // These are immutable, so they can be outside the mutex + project_dir: PathBuf, + ignore_globs: Arc>, +} + +#[derive(Debug)] +pub struct BranchDbInner { + // TODO (Lilith): We need to figure out a way to populate binary docs. + // The old system is hugely flawed and would be best to avoid. + // Instead, consider a helper shim class to track each binary doc as they come in, and update the fs as needed. + // But what's the intended user behavior? Consult with Nikita and Paul. + + // Does this need to be in inner? Probably not, right? + repo: Repo, + username: Option, + + binary_states: HashMap, + branch_states: HashMap>>, + metadata_state: Option<(DocumentId, BranchesMetadataDoc)>, + + // The checked out ref is the ref that the filesystem is currently synced with. + // Has a separate lock because of its importance; it needs to be locked while we're prepping a commit or checking out stuff + checked_out_ref: Arc>>, +} + +impl BranchDb { + pub fn new(repo: Repo, project_dir: PathBuf, ignore_globs: Vec) -> Self { + Self { + project_dir, + ignore_globs: Arc::new(ignore_globs), + inner: Arc::new(Mutex::new(BranchDbInner { + repo, + username: None, + binary_states: HashMap::new(), + branch_states: HashMap::new(), + metadata_state: None, + checked_out_ref: Arc::new(RwLock::new(None)), + })), + } + } + + + pub fn get_ignore_globs(&self) -> Vec { + (*self.ignore_globs).clone() + } + + pub fn get_project_dir(&self) -> PathBuf { + self.project_dir.clone() + } + + pub async fn set_username(&self, username: Option) { + let mut inner = self.inner.lock().await; + inner.username = username; + } + + /// Get the mutable checked out ref for locking. + pub async fn get_checked_out_ref_mut(&self) -> Arc>> { + return self.inner.lock().await.checked_out_ref.clone(); + } + + pub async fn get_metadata_state(&self) -> Option<(DocumentId, BranchesMetadataDoc)> { + let inner = self.inner.lock().await; + // This is a needlessly expensive operation; we should consider allowing reference introspection via external lockers. + // And/or improve clone perf by reducing string usage in BranchesMetadataDoc. + inner.metadata_state.clone() + } + + pub async fn set_metadata_state(&self, id: DocumentId, state: BranchesMetadataDoc) { + let mut inner = self.inner.lock().await; + inner.metadata_state = Some((id, state)); + } + + pub async fn has_branch(&self, id: &DocumentId) -> bool { + let inner = self.inner.lock().await; + return inner.branch_states.contains_key(id); + } + + pub async fn insert_branch_state_if_not_exists(&self, id: DocumentId, f: F) + where + F: FnOnce() -> BranchState, + { + let mut inner = self.inner.lock().await; + inner + .branch_states + .entry(id.clone()) + .or_insert_with(|| Arc::new(Mutex::new(f()))); + } + + // This exposes inner BranchState objects via Arc. This is important because we use branch states all over the place. + // Alternatively we could provide a read-only view in a closure, or clone them. + // We do need to be a little careful about locks though. + pub async fn get_branch_state(&self, id: &DocumentId) -> Option>> { + let inner = self.inner.lock().await; + inner.branch_states.get(id).cloned() + } + + pub async fn get_branch_handle(&self, id: &DocumentId) -> Option { + let inner = self.inner.lock().await; + let Some(state) = inner.branch_states.get(id) else { + return None; + }; + Some(state.lock().await.doc_handle.clone()) + } +} diff --git a/rust/src/project/branch_db/branch.rs b/rust/src/project/branch_db/branch.rs new file mode 100644 index 00000000..394cc1cd --- /dev/null +++ b/rust/src/project/branch_db/branch.rs @@ -0,0 +1,99 @@ +use std::collections::HashMap; + +use automerge::Automerge; +use autosurgeon::reconcile; +use samod::DocHandle; + +use crate::{ + helpers::{ + branch::{Branch, BranchesMetadataDoc, GodotProjectDoc}, + utils::{CommitMetadata, commit_with_attribution_and_timestamp}, + }, + project::branch_db::BranchDb, +}; + +// Methods related to branch and document management on a [BranchDb]. +impl BranchDb { + /// Create a new metadata document, and a new main branch, and return the handle of the metadata document. + pub async fn create_metadata_doc(&self) -> DocHandle { + let repo = self.inner.lock().await.repo.clone(); + let username = self.inner.lock().await.username.clone(); + + // Create new main branch doc + let main_handle = repo.create(Automerge::new()).await.unwrap(); + let main_handle_clone = main_handle.clone(); + let username_clone = username.clone(); + + tokio::task::spawn_blocking(move || { + main_handle_clone.with_document(|d| { + let mut tx = d.transaction(); + let _ = reconcile( + &mut tx, + GodotProjectDoc { + files: HashMap::new(), + state: HashMap::new(), + }, + ); + commit_with_attribution_and_timestamp( + tx, + &CommitMetadata { + username: username_clone, + branch_id: Some(main_handle_clone.document_id().clone()), + merge_metadata: None, + reverted_to: None, + changed_files: None, + is_setup: Some(true), + }, + ); + }); + }) + .await + .unwrap(); + + let main_branch_doc_id = main_handle.document_id().to_string(); + let main_branch_doc_id_clone = main_branch_doc_id.clone(); + let branches = HashMap::from([( + main_branch_doc_id, + Branch { + name: String::from("main"), + id: main_handle.document_id().to_string(), + fork_info: None, + merge_info: None, + created_by: username.clone(), + merged_into: None, + reverted_to: None, + }, + )]); + let branches_clone = branches.clone(); + + // create new branches metadata doc + let metadata_handle = repo.create(Automerge::new()).await.unwrap(); + let metadata_handle_clone = metadata_handle.clone(); + tokio::task::spawn_blocking(move || { + metadata_handle.with_document(|d| { + let mut tx = d.transaction(); + let _ = reconcile( + &mut tx, + BranchesMetadataDoc { + main_doc_id: main_branch_doc_id_clone, + branches: branches_clone, + }, + ); + commit_with_attribution_and_timestamp( + tx, + &CommitMetadata { + username: username, + branch_id: None, + merge_metadata: None, + reverted_to: None, + changed_files: None, + is_setup: Some(true), + }, + ); + }); + }) + .await + .unwrap(); + metadata_handle_clone + } +} diff --git a/rust/src/project/branch_db/commit.rs b/rust/src/project/branch_db/commit.rs new file mode 100644 index 00000000..d6d8e83f --- /dev/null +++ b/rust/src/project/branch_db/commit.rs @@ -0,0 +1,254 @@ +use std::collections::{HashMap, HashSet}; + +use automerge::{Automerge, ChangeHash, ObjId, ObjType, ROOT, ReadDoc}; +use autosurgeon::Doc; +use samod::{DocHandle, DocumentId}; + +use crate::{ + fs::file_utils::FileContent, + helpers::{ + doc_utils::SimpleDocReader, + utils::{ + ChangeType, ChangedFile, CommitMetadata, commit_with_attribution_and_timestamp, + heads_to_vec_string, + }, + }, + parser::godot_parser::GodotScene, + project::branch_db::{BranchDb, HistoryRef}, +}; + +// Methods related to committing changes to a branch in [BranchDb]. +impl BranchDb { + /// Commit a list of files from the filesystem, while ensuring they've actually been changed before including them. + pub async fn commit_fs_changes( + &self, + files: Vec<(String, FileContent)>, + ref_: &HistoryRef, + revert: Option>, + is_checking_in: bool, + ) -> Option { + let Some(branch_handle) = self.get_branch_handle(&ref_.branch).await else { + return None; + }; + + // Only commit files that have actually changed + let files = self.filter_changed_files(ref_, files).await; + let username = self.inner.lock().await.username.clone(); + + let mut binary_entries: Vec<(String, DocHandle)> = Vec::new(); + let mut text_entries: Vec<(String, String)> = Vec::new(); + let mut scene_entries: Vec<(String, GodotScene)> = Vec::new(); + let mut deleted_entries: Vec = Vec::new(); + + for (path, content) in files { + match content { + FileContent::Binary(content) => { + let handle = self.create_new_binary_doc(content).await; + binary_entries.push((path, handle)); + } + FileContent::String(content) => { + text_entries.push((path, content)); + } + FileContent::Scene(godot_scene) => { + scene_entries.push((path, godot_scene)); + } + FileContent::Deleted => { + deleted_entries.push(path); + } + } + } + + let new_heads = tokio::task::spawn_blocking(move || { + branch_handle.with_document(|d| { + // We currently only ever save files to the current heads + let mut tx = d.transaction(); + + let mut changes: Vec = Vec::new(); + let files = tx.get_obj_id(ROOT, "files").unwrap(); + + // write text entries to doc + for (path, content) in text_entries { + // get existing file url or create new one + let (file_entry, change_type) = match tx.get(&files, &path) { + Ok(Some((automerge::Value::Object(ObjType::Map), file_entry))) => { + (file_entry, ChangeType::Modified) + } + _ => ( + tx.put_object(&files, &path, ObjType::Map).unwrap(), + ChangeType::Added, + ), + }; + + changes.push(ChangedFile { path, change_type }); + + // delete url in file entry if it previously had one + if let Ok(Some((_, _))) = tx.get(&file_entry, "url") { + let _ = tx.delete(&file_entry, "url"); + } + + // delete structured content in file entry if it previously had one + if let Ok(Some((_, _))) = tx.get(&file_entry, "structured_content") { + let _ = tx.delete(&file_entry, "structured_content"); + } + + // either get existing text or create new text + let content_key = match tx.get(&file_entry, "content") { + Ok(Some((automerge::Value::Object(ObjType::Text), content))) => content, + _ => tx + .put_object(&file_entry, "content", ObjType::Text) + .unwrap(), + }; + let _ = tx.update_text(&content_key, &content); + } + + // write scene entries to doc + for (path, godot_scene) in scene_entries { + // get the change flag + let change_type = match tx.get(&files, &path) { + Ok(Some(_)) => ChangeType::Modified, + _ => ChangeType::Added, + }; + + let scene_file = tx + .get_obj_id(&files, &path) + .unwrap_or_else(|| tx.put_object(&files, &path, ObjType::Map).unwrap()); + autosurgeon::reconcile_prop( + &mut tx, + &scene_file, + "structured_content", + godot_scene, + ) + .unwrap_or_else(|e| { + tracing::error!("error reconciling scene: {}", e); + panic!("error reconciling scene: {}", e); + }); + changes.push(ChangedFile { path, change_type }); + } + + // write binary entries to doc + for (path, binary_doc_handle) in binary_entries { + // get the change flag + let change_type = match tx.get(&files, &path) { + Ok(Some(_)) => ChangeType::Modified, + _ => ChangeType::Added, + }; + + let file_entry = tx.put_object(&files, &path, ObjType::Map); + let _ = tx.put( + file_entry.unwrap(), + "url", + format!("automerge:{}", &binary_doc_handle.document_id()), + ); + + changes.push(ChangedFile { path, change_type }); + } + + for path in deleted_entries { + let _ = tx.delete(&files, &path); + changes.push(ChangedFile { + path, + change_type: ChangeType::Removed, + }); + } + + commit_with_attribution_and_timestamp( + tx, + &CommitMetadata { + username: username.clone(), + branch_id: Some(branch_handle.document_id().clone()), + merge_metadata: None, + reverted_to: match revert { + Some(revert) => Some(heads_to_vec_string(revert)), + None => None, + }, + changed_files: Some(changes), + is_setup: Some(is_checking_in), + }, + ); + + // TODO: I actually have no idea if this works -- do we need a new with_document + // to check for new heads? + d.get_heads() + }) + }) + .await + .unwrap(); + + assert!(new_heads != ref_.heads); + return Some(HistoryRef { + heads: new_heads, + branch: ref_.branch.clone(), + }); + } + + // Filter a list of files to those changed compared to a given ref. + async fn filter_changed_files( + &self, + ref_: &HistoryRef, + files: Vec<(String, FileContent)>, + ) -> Vec<(String, FileContent)> { + // Only load files matching those we've provided + let filter = files + .iter() + .map(|(path, _)| path.to_string()) + .collect::>(); + + // Check our stored files + let stored_files = self + .get_files_at_ref(&ref_, &filter) + .await + .unwrap_or(HashMap::new()); + + // Filter out files that haven't actually changed + files + .into_iter() + .filter_map(|(path, content)| { + let path = path.to_string(); + let stored_content = stored_files.get(&path); + if let Some(stored_content) = stored_content { + if stored_content == &content { + return None; + } + } + Some((path, content)) + }) + .collect() + } + + + pub async fn create_new_binary_doc(&self, content: Vec) -> DocHandle { + let handle = self + .inner + .lock() + .await + .repo + .create(Automerge::new()) + .await + .unwrap(); + + let username = self.inner.lock().await.username.clone(); + + // we're allowed to transact in the background: nobody needs this to exist yet. + let h = handle.clone(); + tokio::task::spawn_blocking(move || { + h.with_document(|d| { + let mut tx = d.transaction(); + let _ = tx.put(ROOT, "content", content); + commit_with_attribution_and_timestamp( + tx, + &CommitMetadata { + username: username, + branch_id: None, + merge_metadata: None, + reverted_to: None, + changed_files: None, + is_setup: Some(false), + }, + ); + }); + }); + + // TODO: actually store the handle + return handle; + } +} diff --git a/rust/src/project/branch_db/file.rs b/rust/src/project/branch_db/file.rs new file mode 100644 index 00000000..d77cfc10 --- /dev/null +++ b/rust/src/project/branch_db/file.rs @@ -0,0 +1,284 @@ +use std::{collections::{HashMap, HashSet}, path::PathBuf}; + +use automerge::{ObjId, ObjType, ROOT, ReadDoc}; +use samod::DocumentId; + +use crate::{ + fs::{file_utils::FileSystemEvent, file_utils::FileContent}, + helpers::{doc_utils::SimpleDocReader, utils::get_changed_files}, + project::branch_db::{BranchDb, HistoryRef}, +}; + +/// Methods related to getting file changes and file contents out of documents. +impl BranchDb { + // Utility to check for shared history between refs + async fn shares_history(&self, earlier_ref: HistoryRef, later_ref: HistoryRef) -> bool { + let Some(handle) = self.get_branch_handle(&later_ref.branch).await else { + return false; + }; + tokio::task::spawn_blocking(move || { + handle.with_document(move |d| { + d.get_obj_id_at(ROOT, "files", &earlier_ref.heads).is_some() + && d.get_obj_id_at(ROOT, "files", &later_ref.heads).is_some() + }) + }) + .await + .unwrap() + } + + /// Given two refs, checks to see if one is a direct descendant of another. + /// If it is, it returns the more up-to-date ref. + /// If not, it returns None. + async fn get_descendent_ref( + &self, + ref_a: &HistoryRef, + ref_b: &HistoryRef, + ) -> Option { + // If we can't compare them, they can't share a history + if !ref_a.is_valid() || !ref_b.is_valid() { + return None; + } + if self.shares_history(ref_a.clone(), ref_b.clone()).await { + return Some(ref_b.clone()); + } + if self.shares_history(ref_b.clone(), ref_a.clone()).await { + return Some(ref_a.clone()); + } + None + } + // TODO (Lilith): During profiling, look at this method. It seems quite improvable. + // Here's my idea: + // In each branch doc, store an md5 hash of the file contents. + // Then, we can get a vector of changed files and their operations by comparing the two tracked_files + // using a single with_document call. With that, we can construct a filter to make the hydration lighter. + // That would significantly improve the slow diff. I'm not sure if that would be faster than the fast diff. + // (But if they're equivalent, not dealing with patches significantly simplifies code.) + + // Afterwards, another possible improvement here: + // Instead of fetching the file content, we could just get the changed files. + // Then, later code could fetch the file content asynchronously. + + /// Get a list of file operations between two points in Patchwork history. + /// If one ref exists in the history of another, we can do a fast automerge diff. + /// If they have diverged, we must do a slow file-wise diff. + pub async fn get_changed_file_content_between_refs( + &self, + old_ref: Option<&HistoryRef>, + new_ref: &HistoryRef, + force_slow_diff: bool, + ) -> Option> { + if !new_ref.is_valid() { + tracing::warn!("new ref is empty, can't get changed files"); + return None; + } + + if old_ref.is_none() || !old_ref.unwrap().is_valid() { + tracing::info!("old heads empty, getting ALL files on branch"); + + let files = self.get_files_at_ref(&new_ref, &HashSet::new()).await?; + + return Some( + files + .into_iter() + .map(|(path, content)| match content { + FileContent::Deleted => FileSystemEvent::FileDeleted(PathBuf::from(path)), + _ => FileSystemEvent::FileCreated(PathBuf::from(path), content), + }) + .collect(), + ); + } + + let old_ref = old_ref.unwrap(); + + let descendent_ref = self.get_descendent_ref(old_ref, new_ref).await; + + if descendent_ref.is_none() || force_slow_diff { + // neither document is the descendent of the other, we can't do a fast diff, + // we need to do it the slow way; get the files from both docs + let old_files = self.get_files_at_ref(old_ref, &HashSet::new()).await?; + let new_files = self.get_files_at_ref(new_ref, &HashSet::new()).await?; + + let mut events = Vec::new(); + for (path, _) in old_files.iter() { + if !new_files.contains_key(path) { + events.push(FileSystemEvent::FileDeleted(PathBuf::from(path))); + } + } + for (path, content) in new_files { + match content { + FileContent::Deleted => { + events.push(FileSystemEvent::FileDeleted(PathBuf::from(path))); + continue; + } + _ => {} + } + if !old_files.contains_key(&path) { + events.push(FileSystemEvent::FileCreated(PathBuf::from(path), content)); + } else if &content != old_files.get(&path).unwrap() { + events.push(FileSystemEvent::FileModified(PathBuf::from(path), content)); + } + } + return Some(events); + } + + let descendent_ref = descendent_ref.unwrap(); + let handle = self.get_branch_handle(&descendent_ref.branch).await?; + + // Get the patches from the later (descendant) ref + let old_heads = old_ref.heads.clone(); + let new_heads = new_ref.heads.clone(); + let (patches, old_file_set, curr_file_set) = tokio::task::spawn_blocking(move || { + handle.with_document(|d| { + let old_files_id: Option = d.get_obj_id_at(ROOT, "files", &old_heads); + let curr_files_id = d.get_obj_id_at(ROOT, "files", &new_heads); + let old_file_set = if old_files_id.is_none() { + HashSet::::new() + } else { + d.keys_at(&old_files_id.unwrap(), &old_heads) + .into_iter() + .collect::>() + }; + let curr_file_set = if curr_files_id.is_none() { + HashSet::::new() + } else { + d.keys_at(&curr_files_id.unwrap(), &new_heads) + .into_iter() + .collect::>() + }; + let patches = d.diff(&old_heads, &new_heads); + (patches, old_file_set, curr_file_set) + }) + }) + .await + .unwrap(); + + // Gather the information of what files changed from the patches. + let deleted_files: HashSet<_> = old_file_set.difference(&curr_file_set).cloned().collect(); + let added_files: HashSet<_> = curr_file_set.difference(&old_file_set).cloned().collect(); + let modified_files: HashSet<_> = get_changed_files(&patches) + .into_iter() + .filter(|f| !deleted_files.contains(f)) + .filter(|f| !added_files.contains(f)) + .collect(); + let all_files: HashSet<_> = deleted_files + .iter() + .chain(added_files.iter()) + .chain(modified_files.iter()) + .cloned() + .collect(); + + // Get the files, then convert them into events using the information we gathered. + Some( + self.get_files_at_ref(new_ref, &all_files) + .await? + .into_iter() + .map(|(path, content)| match content { + FileContent::Deleted => FileSystemEvent::FileDeleted(PathBuf::from(path)), + _ if added_files.contains(&path) => { + FileSystemEvent::FileCreated(PathBuf::from(path), content) + } + _ if deleted_files.contains(&path) => { + FileSystemEvent::FileDeleted(PathBuf::from(path)) + } + _ => FileSystemEvent::FileModified(PathBuf::from(path), content), + }) + .chain( + deleted_files + .iter() + .map(|path| FileSystemEvent::FileDeleted(PathBuf::from(path))), + ) + .collect(), + ) + } + + async fn get_linked_file(&self, doc_id: &DocumentId) -> Option { + let state = self.inner.lock().await.binary_states.get(doc_id).cloned(); + let Some(handle) = state.and_then(|f| f.doc_handle) else { + return None; + }; + tokio::task::spawn_blocking(move || { + handle.with_document(|d| match d.get(ROOT, "content") { + Ok(Some((value, _))) if value.is_bytes() => { + Some(FileContent::Binary(value.into_bytes().unwrap())) + } + Ok(Some((value, _))) if value.is_str() => { + Some(FileContent::String(value.into_string().unwrap())) + } + _ => None, + }) + }) + .await + .unwrap() + } + + pub async fn get_files_at_ref( + &self, + desired_ref: &HistoryRef, + filters: &HashSet, + ) -> Option> { + let Some(branch_state) = self.get_branch_state(&desired_ref.branch).await else { + return None; + }; + + let mut files = HashMap::new(); + let mut linked_doc_ids = Vec::new(); + + let doc_handle = branch_state.lock().await.doc_handle.clone(); + let filters = filters.clone(); + let desired_ref = desired_ref.clone(); + let (mut files, linked_doc_ids) = tokio::task::spawn_blocking(move || { + doc_handle.with_document(|doc| { + let files_obj_id: ObjId = doc + .get_at(ROOT, "files", desired_ref.heads.as_ref()) + .unwrap() + .unwrap() + .1; + for path in doc.keys_at(&files_obj_id, desired_ref.heads.as_ref()) { + if !filters.is_empty() && !filters.contains(&path) { + continue; + } + let file_entry = + match doc.get_at(&files_obj_id, &path, desired_ref.heads.as_ref()) { + Ok(Some((automerge::Value::Object(ObjType::Map), file_entry))) => { + file_entry + } + _ => panic!("failed to get file entry for {:?}", path), + }; + + match FileContent::hydrate_content_at( + file_entry, + &doc, + &path, + desired_ref.heads.as_ref(), + ) { + Ok(content) => { + files.insert(path, content); + } + Err(res) => match res { + Ok(id) => { + linked_doc_ids.push((id, path)); + } + Err(error_msg) => { + tracing::error!("error: {:?}", error_msg); + } + }, + }; + } + }); + (files, linked_doc_ids) + }) + .await + .unwrap(); + + for (doc_id, path) in linked_doc_ids { + let linked_file_content: Option = self.get_linked_file(&doc_id).await; + if let Some(file_content) = linked_file_content { + files.insert(path, file_content); + } else { + tracing::warn!("linked file {:?} not found", path); + } + } + + return Some(files); + } +} diff --git a/rust/src/project/branch_db/util.rs b/rust/src/project/branch_db/util.rs new file mode 100644 index 00000000..bf3c5d28 --- /dev/null +++ b/rust/src/project/branch_db/util.rs @@ -0,0 +1,79 @@ +use std::{path::PathBuf, str::FromStr}; + +use samod::DocumentId; + +use crate::project::branch_db::{BranchDb, HistoryRef}; + +// Utility methods for working with [BranchDb]. +impl BranchDb { + /// Turns a filesytem path into a project-local res:// path. + pub fn localize_path(&self, path: &PathBuf) -> String { + let path = path.to_string_lossy().replace("\\", "/"); + let project_dir = self.project_dir.to_string_lossy().replace("\\", "/"); + if path.starts_with(&project_dir) { + // TODO: this isn't teeechnically a Path, it's a URL... PathBuf is probably the wrong choice + let thing = PathBuf::from("res://".to_string()) + .join(PathBuf::from(&path[project_dir.len()..].to_string())); + thing.to_string_lossy().to_string() + } else { + path.to_string() + } + } + + /// Convert a project URL like res:// into a local filesystem path. + pub fn globalize_path(&self, path: &String) -> PathBuf { + // trim the project_dir from the front of the path + if path.starts_with("res://") { + self.project_dir.clone().join(&path["res://".len()..]) + } else { + PathBuf::from(path) + } + } + + /// Get the most recent ref on a given branch. + // TODO (Lilith): This replaces branch_state.synced_heads. Either remove synced_heads, + // or figure out a way to reliably update it when the heads actually change. + // In the old system, synced heads was just force-updated every branch update. + // Maybe that's enough? Get DocumentWatcher to do it? Then we remove the with_doc call here. + pub async fn get_latest_ref_on_branch(&self, branch: &DocumentId) -> Option { + let state = self.get_branch_state(branch).await; + let Some(state) = state else { + tracing::error!("Couldn't get latest ref on branch; branch state not loaded!"); + return None; + }; + let state = state.lock().await; + let handle = state.doc_handle.clone(); + let heads = tokio::task::spawn_blocking(move || handle.with_document(|d| d.get_heads())).await.unwrap(); + + Some(HistoryRef { + heads, + branch: branch.clone() + }) + } + + pub async fn get_main_branch(&self) -> Option { + let Some((_, metadata)) = self.get_metadata_state().await else { + tracing::error!("Couldn't get main branch; no metadata doc."); + return None; + }; + // TODO (Lilith): Figure out a way to hydrate/reconcile DocumentID so we don't have to do the string parse here. + // Alternatively, don't store BranchesMetadataDoc, store some similar thing to BranchState + return DocumentId::from_str(&metadata.main_doc_id).ok(); + } + + /// Check if a path should be ignored based on the provided glob patterns + pub fn should_ignore(&self, path: &PathBuf) -> bool { + // TODO: We should check if it's a symlink or not, but right now it's sufficient to just check if it's outside of the watch path + // check if it's outside of the watch path + if path.is_symlink() { + return true; + } + if !path.starts_with(&self.project_dir) { + return true; + } + self.ignore_globs + .iter() + .any(|pattern| pattern.matches(&path.to_string_lossy())) + } + +} \ No newline at end of file diff --git a/rust/src/project/connection.rs b/rust/src/project/connection.rs new file mode 100644 index 00000000..e1191132 --- /dev/null +++ b/rust/src/project/connection.rs @@ -0,0 +1,162 @@ +use futures::{Stream, StreamExt as _}; +use samod::{ConnDirection, ConnFinishedReason, Repo}; +use tokio::{ + net::TcpStream, + sync::{broadcast, watch}, + task::JoinHandle, +}; +use tokio_stream::wrappers::BroadcastStream; + +#[derive(Debug, Clone)] +enum ConnectionStoppedReason { + TcpConnectionError(String), + WebSocketsConnectionError(String), + ConnectionCompleted(ConnFinishedReason), +} + +#[derive(Debug, Clone)] +pub enum RemoteConnectionEvent { + ConnectionFailed, + Connected, + ConnectionCompleted, +} + +#[derive(Debug, Clone)] +pub enum RemoteConnectionStatus { + Connected, + Disconnected, +} + +/// Connects a repo to the remote server. +#[derive(Debug)] +pub struct RemoteConnection { + connection_handle: JoinHandle<()>, + status_rx: watch::Receiver, + events_tx: broadcast::Sender, +} + +#[derive(Clone, Debug)] +struct RemoteConnectionInner { + repo: Repo, + server_url: String, + events_tx: broadcast::Sender, + status_tx: watch::Sender, +} + +impl Drop for RemoteConnection { + // Stop the connection on drop + fn drop(&mut self) { + &self.connection_handle.abort(); + } +} + +impl RemoteConnection { + /// Starts a connection to the server. A background task will be dispatched that + /// reattempts connection indefinitely until the handle is dropped. + pub fn new(repo: Repo, server_url: String) -> Self { + let (events_tx, _) = broadcast::channel(32); + let (status_tx, status_rx) = watch::channel(RemoteConnectionStatus::Disconnected); + let inner = RemoteConnectionInner { + repo, + server_url, + events_tx: events_tx.clone(), + status_tx, + }; + + let connection_handle: JoinHandle<()> = tokio::spawn(async move { + inner.retry_connection().await; + }); + + Self { + connection_handle, + status_rx, + events_tx, + } + } + + /// Subscribe to future events. + pub fn events(&self) -> impl Stream { + let rx = self.events_tx.subscribe(); + BroadcastStream::new(rx).filter_map(|result| async move { + match result { + Ok(event) => Some(event), + // Happens when the stream lags + Err(err) => { + tracing::warn!("Dropped remote connection events: {:?}", err); + None + } + } + }) + } + + /// Get the current status of the remote connection. + pub fn status(&self) -> RemoteConnectionStatus { + self.status_rx.borrow().clone() + } +} + +impl RemoteConnectionInner { + async fn retry_connection(&self) { + // The old code responded to different connection stop reason to change the backoff behavior. + // If we want to keep doing that, add intelligent backoff here. + // For now, this just tries once every second forever and ever. + let backoff = 1000; + loop { + let termination_reason = self.try_connection().await; + tracing::error!("Connection failure: {:?}", termination_reason); + tracing::error!("Retrying in {}ms...", backoff); + tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await; + } + } + + async fn try_connection(&self) -> ConnectionStoppedReason { + let repo_handle = self.repo.clone(); + let server_url = self.server_url.clone(); + + tracing::info!("Attempting to connect to server at {server_url}..."); + + // Connect via websockets + let connection = if server_url.starts_with("ws://") { + let res = tokio_tungstenite::connect_async(server_url.clone()).await; + match res { + Err(e) => { + self.events_tx.send(RemoteConnectionEvent::ConnectionFailed); + return ConnectionStoppedReason::WebSocketsConnectionError(e.to_string()); + } + Ok((res, _)) => repo_handle + .connect_tungstenite(res, ConnDirection::Outgoing) + .unwrap(), + } + } + // Connect via TCP + else { + let res = TcpStream::connect(server_url.clone()).await; + match res { + Err(e) => { + self.events_tx.send(RemoteConnectionEvent::ConnectionFailed); + return ConnectionStoppedReason::TcpConnectionError(e.to_string()); + } + Ok(res) => repo_handle + .connect_tokio_io(res, ConnDirection::Outgoing) + .unwrap(), + } + }; + + tracing::info!("Connected successfully!"); + + if let Err(e) = connection.handshake_complete().await { + self.status_tx.send(RemoteConnectionStatus::Disconnected); + self.events_tx.send(RemoteConnectionEvent::ConnectionFailed); + return ConnectionStoppedReason::ConnectionCompleted(e); + } + + tracing::info!("Handshake completed!"); + self.status_tx.send(RemoteConnectionStatus::Connected); + self.events_tx.send(RemoteConnectionEvent::Connected); + + let completed = connection.finished().await; + self.status_tx.send(RemoteConnectionStatus::Disconnected); + self.events_tx.send(RemoteConnectionEvent::ConnectionCompleted); + return ConnectionStoppedReason::ConnectionCompleted(completed); + } +} diff --git a/rust/src/project/document_watcher.rs b/rust/src/project/document_watcher.rs new file mode 100644 index 00000000..8c9cd0ac --- /dev/null +++ b/rust/src/project/document_watcher.rs @@ -0,0 +1,231 @@ +use std::{ + collections::{HashMap, HashSet}, + str::FromStr, + sync::Arc, +}; + +use crate::{ + helpers::{ + branch::{ + BranchState, BranchStateForkInfo, BranchStateMergeInfo, BranchStateRevertInfo, + BranchesMetadataDoc, + }, + doc_utils::SimpleDocReader, + utils::parse_automerge_url, + }, + project::branch_db::BranchDb, +}; +use automerge::{ChangeHash, ROOT, ReadDoc}; +use autosurgeon::hydrate; +use futures::{FutureExt, StreamExt}; +use samod::{DocHandle, DocumentId, Repo}; +use tokio::{sync::Mutex, task::JoinHandle}; + +/// Tracks branch and metadata documents from an Automerge repo, updating BranchDB when the state changes. +#[derive(Clone, Debug)] +pub struct DocumentWatcher { + repo: Repo, + branch_db: BranchDb, + tracking_handles: Arc>>>, +} + +impl Drop for DocumentWatcher { + fn drop(&mut self) { + for handle in self.tracking_handles.blocking_lock().drain(..) { + // NOTE: spawn_blocking calls are not immediately aborted. They will run to completion in the background, then finish. + handle.abort(); + } + } +} + +impl DocumentWatcher { + /// Spawns the [DocumentWatcher], creating parallel tasks for the metadata document tracking and subsequent tasks for any child documents. + pub fn new(repo: Repo, branch_db: BranchDb, metadata_handle: DocHandle) -> Self { + let this = Self { + repo, + branch_db, + tracking_handles: Arc::new(Mutex::new(Vec::new())), + }; + + let this_clone = this.clone(); + + this.tracking_handles + .blocking_lock() + .push(tokio::spawn(async move { + // do the initial ingest + this_clone + .ingest_metadata_document(metadata_handle.clone()) + .await; + // track changes for future ingests + this_clone.track_metadata_document(metadata_handle).await; + })); + + return this; + } + + // The branch documents are a document for each branch, containing all the serialized data for all scenes and text files. + async fn track_branch_document(&self, handle: DocHandle) { + let mut stream = handle.changes(); + loop { + let _ = stream.next().await; + // collapse the rest of the stream, in case multiple futures are ready + while let Some(_) = stream.next().now_or_never().flatten() {} + self.ingest_branch_document(handle.clone()).await; + } + } + + // The metadata document is the root document containing IDs of all branch docs. + async fn track_metadata_document(&self, handle: DocHandle) { + let mut stream = handle.changes(); + loop { + let _ = stream.next().await; + // collapse the rest of the stream, in case multiple futures are ready + while let Some(_) = stream.next().now_or_never().flatten() {} + self.ingest_metadata_document(handle.clone()).await; + } + } + + async fn ingest_branch_document(&self, handle: DocHandle) { + let (_, meta) = self.branch_db.get_metadata_state().await.expect( + "Somehow, we haven't loaded a metadata doc, but we're ingesting a branch document?!?!", + ); + + let branch = meta + .branches + .get(&handle.document_id().to_string()) + .unwrap(); + + // Create a default branch state, but only if we don't have an existing branch state. + let h = handle.clone(); + self.branch_db + .insert_branch_state_if_not_exists(handle.document_id().clone(), move || BranchState { + name: branch.name.clone(), + doc_handle: h.clone(), + linked_doc_ids: HashSet::new(), + synced_heads: Vec::new(), + fork_info: match &branch.fork_info { + Some(fork_info) => Some(BranchStateForkInfo { + forked_from: DocumentId::from_str(&fork_info.forked_from).unwrap(), + forked_at: fork_info + .forked_at + .iter() + .map(|h| ChangeHash::from_str(h).unwrap()) + .collect(), + }), + None => None, + }, + merge_info: match &branch.merge_info { + Some(merge_info) => Some(BranchStateMergeInfo { + merge_into: DocumentId::from_str(&merge_info.merge_into).unwrap(), + merge_at: merge_info + .merge_at + .iter() + .map(|h| ChangeHash::from_str(h).unwrap()) + .collect(), + }), + None => None, + }, + is_main: h.document_id().to_string() == meta.main_doc_id, + created_by: branch.created_by.clone(), + merged_into: match &branch.merged_into { + Some(merged_into) => match DocumentId::from_str(&merged_into) { + Ok(merged_into) => Some(merged_into), + Err(_) => None, + }, + None => None, + }, + revert_info: match &branch.reverted_to { + Some(reverted_to) => Some(BranchStateRevertInfo { + reverted_to: reverted_to + .iter() + .map(|h| ChangeHash::from_str(h).unwrap()) + .collect(), + }), + None => None, + }, + }) + .await; + + let h = handle.clone(); + let linked_docs = tokio::task::spawn_blocking(move || { + // Collect all linked doc IDs from this branch + h.with_document(|d| { + let files = match d.get_obj_id(ROOT, "files") { + Some(files) => files, + None => { + tracing::warn!("Failed to load files for branch doc {:?}", h.document_id()); + return HashMap::new(); + } + }; + + d.keys(&files) + .filter_map(|path| { + let file = match d.get_obj_id(&files, &path) { + Some(file) => file, + None => { + tracing::error!("Failed to load linked doc {:?}", path); + return None; + } + }; + + let url = match d.get_string(&file, "url") { + Some(url) => url, + None => { + return None; + } + }; + + parse_automerge_url(&url).map(|id| (path.clone(), id)) + }) + .collect::>() + }) + }) + .await + .unwrap(); + + let branch_state_mutex = self + .branch_db + .get_branch_state(&handle.document_id()) + .await + .unwrap(); + let mut branch_state = branch_state_mutex.lock().await; + branch_state.linked_doc_ids = linked_docs.values().cloned().collect(); + } + + async fn ingest_metadata_document(&self, handle: DocHandle) { + // TODO: Stop tracking removed branches + // Find added branches, and begin tracking them + let h = handle.clone(); + let meta = tokio::task::spawn_blocking(move || { + // TODO: correct error handling on hydration failure; currently panics! + let branches_metadata: BranchesMetadataDoc = h.with_document(|d| hydrate(d).unwrap()); + branches_metadata + }) + .await + .unwrap(); + self.branch_db + .set_metadata_state(handle.document_id().clone(), meta.clone()); + // check if there are new branches that haven't loaded yet + for (branch_id_str, _) in meta.branches.iter() { + let branch_id = DocumentId::from_str(branch_id_str).unwrap(); + + if !self.branch_db.has_branch(&branch_id).await { + let Some(handle) = self.repo.find(branch_id.clone()).await.unwrap() else { + tracing::error!( + "Document {:?} exists in the branch metadata document, but not the repo! Skipping.", + branch_id + ); + continue; + }; + self.ingest_branch_document(handle.clone()).await; + // Track the document + let this = self.clone(); + self.tracking_handles + .blocking_lock() + .push(tokio::spawn(async move { + this.track_branch_document(handle).await + })); + } + } + } +} diff --git a/rust/src/project/driver.rs b/rust/src/project/driver.rs new file mode 100644 index 00000000..5867efa9 --- /dev/null +++ b/rust/src/project/driver.rs @@ -0,0 +1,483 @@ +use crate::diff::differ::ProjectDiff; +use crate::fs::file_utils::FileSystemEvent; +use crate::helpers::branch::BranchState; +use crate::helpers::utils::{CommitInfo, CommitMetadata}; +use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkEditorAccessor}; +use crate::project::branch_db::{BranchDb, HistoryRef}; +use crate::project::connection::{RemoteConnection, RemoteConnectionEvent, RemoteConnectionStatus}; +use crate::project::document_watcher::DocumentWatcher; +use crate::project::peer_watcher::PeerWatcher; +use crate::project::sync_automerge_to_fs::SyncAutomergeToFileSystem; +use crate::project::sync_fs_to_automerge::SyncFileSystemToAutomerge; +use automerge::ChangeHash; +use futures::StreamExt; +use samod::{ConcurrencyConfig, DocHandle, DocumentId, Repo}; +use std::path::PathBuf; +use std::str::FromStr; +use std::sync::Arc; + +/// The main driver for the project. +/// Hooks together all the various controllers. +/// When this object is constructed, it is started. When the handle is dropped, it shuts down. +#[derive(Clone, Debug)] +pub struct Driver { + inner: Arc, +} + +#[derive(Debug)] +pub struct DriverInner { + connection: RemoteConnection, + branch_db: BranchDb, + peer_watcher: PeerWatcher, + document_watcher: DocumentWatcher, + sync_automerge_to_fs: SyncAutomergeToFileSystem, + sync_fs_to_automerge: SyncFileSystemToAutomerge, + // TODO (Lilith): Currently the differ is broken because it can't be sent across threads due to the Variant cache. + // Figure out a way to fix that. One option is maybe a global singleton cache only on one thread? Or just killing Variants in the differ, which is ideal. + // differ: Differ, +} + +impl Driver { + async fn parse_gitignore( + project_dir: &PathBuf, + gitignore_path: &PathBuf, + ) -> Vec { + let mut ignore_globs = Vec::new(); + + let content = match tokio::fs::read_to_string(gitignore_path).await { + Ok(content) => content, + Err(_) => { + tracing::error!("Couldn't read gitignore file at {:?}", gitignore_path); + return ignore_globs; + } + }; + + for line in content.lines() { + // trim any comments and whitespace + let line = line.trim().split('#').next().unwrap_or_default().trim(); + if line.is_empty() { + continue; + } + let mut new_line = if line.starts_with("/") { + line.to_string() + } else { + project_dir.join(line).to_string_lossy().to_string() + }; + let new_line = if new_line.ends_with("/") { + // just remove the trailing slash + new_line.pop(); + new_line + } else { + new_line + }; + match glob::Pattern::new(&new_line) { + Ok(glob) => ignore_globs.push(glob), + Err(e) => tracing::error!( + "Invalid glob while parsing gitignore {:?}! Error: {}", + gitignore_path, + e + ), + } + } + ignore_globs + } + + /// Creates a new instance of [Driver]. + /// Causes tasks to run in the background. To cancel everything, drop the handle. + /// If we couldn't start the driver, [None] is returned. + pub async fn new( + server_url: String, + project_path: PathBuf, + storage_directory: PathBuf, + metadata_id: Option, + ) -> Option { + // TODO (Lilith): ensure we make this work across the ENTIRE program. Initialize it only once, etc. + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .thread_name("GodotProjectDriver: worker thread") + .build() + .unwrap(); + + let _ = runtime.enter(); + let storage = samod::storage::TokioFilesystemStorage::new(storage_directory); + let repo = Repo::build_tokio() + .with_concurrency(ConcurrencyConfig::Threadpool( + rayon::ThreadPoolBuilder::new().build().unwrap(), + )) + .with_storage(storage) + .load() + .await; + + let project_path = PathBuf::from(project_path); + + // Construct the ignore globs, and link in the gitignores + let ignore_globs = vec![ + "**/.DS_Store", + "**/thumbs.db", + "**/desktop.ini", + "**/patchwork.cfg", + "**/addons/patchwork*", + "**/target/*", + "**/.*", + ] + .iter() + .filter_map(|p| glob::Pattern::new(p).ok()) + .chain( + Self::parse_gitignore(&project_path, &project_path.join(".gitignore")) + .await + .into_iter(), + ) + .chain( + Self::parse_gitignore(&project_path, &project_path.join(".patchworkignore")) + .await + .into_iter(), + ) + .chain( + Self::parse_gitignore(&project_path, &project_path.join(".gdignore")) + .await + .into_iter(), + ) + .collect(); + + // Start the connection + let connection = RemoteConnection::new(repo.clone(), server_url); + let branch_db = BranchDb::new(repo.clone(), project_path, ignore_globs); + + let metadata_handle = match metadata_id { + // If we're expecting an existing ID, try and fetch it. + Some(id) => { + let Some(handle) = Self::get_metadata_handle(&repo, &id, &connection).await else { + return None; + }; + handle + } + // If we need to make a new ID, make a doc. + None => branch_db.create_metadata_doc().await, + }; + + // The document watcher will auto-ingest the provided metadata handle. + let document_watcher = + DocumentWatcher::new(repo.clone(), branch_db.clone(), metadata_handle); + let peer_watcher = PeerWatcher::new(repo.clone(), branch_db.clone()); + let sync_automerge_to_fs = SyncAutomergeToFileSystem::new(branch_db.clone()); + let sync_fs_to_automerge = SyncFileSystemToAutomerge::new(branch_db.clone()); + // let differ = Differ::new(branch_db.clone()); + + // At this point, if we loaded an existing project, we may not have checked it out yet. + // We'll discover that while processing updates, and check it out then. + + Some(Driver { + inner: DriverInner { + connection, + branch_db, + peer_watcher, + document_watcher, + sync_automerge_to_fs, + sync_fs_to_automerge, + // differ, + } + .into(), + }) + } + + pub async fn set_username(&self, username: Option) { + self.inner.branch_db.set_username(username).await; + } + + /// If we're connected to the server, returns true. + /// Otherwise, retries the server connection on state change until it is either connected, + /// or we give up, then returns true if success or false if failure. + async fn ensure_server_connection(connection: &RemoteConnection, retries: i32) -> bool { + // We must subscribe to the events stream BEFORE checking the status. + // This is so that between two lines of code, the status doesn't change before we've inited our stream. + let events = connection.events(); + tokio::pin!(events); + match connection.status() { + RemoteConnectionStatus::Connected => true, + RemoteConnectionStatus::Disconnected => { + let mut connected = false; + // try 3 times + for _ in 0..retries { + if let Some(RemoteConnectionEvent::Connected) = events.next().await { + connected = true; + break; + } + } + connected + } + } + } + + async fn get_metadata_handle( + repo: &Repo, + metadata_id: &DocumentId, + connection: &RemoteConnection, + ) -> Option { + // TODO: This is awkward; instead it would be great to create a fake old document and have it updated from the server asynchronously. + // For now we must wait til the server connects, if we don't have the doc ID locally. + // Before we continue, we must acquire a handle to the metadata document. + // There are three cases to handle: + // a: The document exists on the local repository. + // b: The document exists on the server, and not the local repository. + // c: The document doesn't exist at all. + + // First, we check the local repository. + let Ok(metadata_handle) = repo.find(metadata_id.clone()).await else { + tracing::error!("Can't start the driver; the repo was immediately stopped!"); + return None; + }; + + match metadata_handle { + // We found it locally, or the server connected REALLY quickly. + Some(metadata_handle) => Some(metadata_handle), + // We didn't find it locally. Try the server for a bit. + None => { + // If the hang gets annoying when starting, we could set this to 1 to reduce it to a minimum. + if !Self::ensure_server_connection(&connection, 3).await { + tracing::error!( + "Couldn't find the metadata doc handle locally, and the server couldn't connect!" + ); + return None; + } + + // Try again on the server, if we were able to connect + match repo.find(metadata_id.clone()).await { + Ok(Some(handle)) => Some(handle), + Ok(None) => { + tracing::error!( + "Couldn't find the metadata doc handle, even after connecting to the server!" + ); + return None; + } + Err(e) => { + tracing::error!( + "Can't start the driver; the repo was immediately stopped!" + ); + return None; + } + } + } + } + } + + pub fn merge_branch( + &mut self, + source_branch_doc_id: DocumentId, + target_branch_doc_id: DocumentId, + ) { + // TODO + } + + pub fn create_merge_preview_branch_between( + &mut self, + source_branch_doc_id: DocumentId, + target_branch_doc_id: DocumentId, + ) { + // TODO (Lilith) + } + + pub fn create_revert_preview_branch_for( + &mut self, + branch_doc_id: DocumentId, + revert_to: Vec, + ) { + // TODO (Lilith) + } + + pub fn delete_branch(&mut self, branch_doc_id: DocumentId) { + // TODO (Lilith) + } + + pub fn revert_to_heads(&mut self, to_revert_to: Vec) { + // TODO (Lilith) + // let branch_state = self.get_checked_out_branch_state().unwrap(); + // let heads = branch_state.doc_handle.with_document(|d| { + // d.get_heads() + // }); + // let content = self.get_changed_file_content_between(Some(branch_state.doc_handle.document_id().clone()), branch_state.doc_handle.document_id().clone(), heads.clone(), to_revert_to.clone(), true); + // let files = content.into_iter().map(|event| { + // match event { + // FileSystemEvent::FileCreated(path, content) => (path, content), + // FileSystemEvent::FileModified(path, content) => (path, content), + // FileSystemEvent::FileDeleted(path) => (path, FileContent::Deleted), + // } + // }).collect::>(); + // self.sync_files_at(branch_state.doc_handle.clone(), files, Some(heads), Some(to_revert_to), false); + // self.checked_out_branch_state = CheckedOutBranchState::CheckingOut(branch_state.doc_handle.document_id().clone(), None); + } + + pub async fn get_diff(&self, before: &HistoryRef, after: &HistoryRef) -> ProjectDiff { + // self.differ.get_diff(before, after).await + ProjectDiff::default() + } + + pub async fn get_metadata_doc(&self) -> Option { + self.inner + .branch_db + .get_metadata_state() + .await + .map(|(id, _)| id) + } + + pub fn safe_to_update_godot() -> bool { + return !(EditorFilesystemAccessor::is_scanning() + || PatchworkEditorAccessor::is_editor_importing() + || PatchworkEditorAccessor::is_changing_scene() + || PatchworkEditorAccessor::unsaved_files_open()); + } + + pub async fn get_main_branch(&self) -> Option { + self.inner + .branch_db + .get_metadata_state() + .await + .map(|(_, doc)| DocumentId::from_str(&doc.main_doc_id).unwrap()) + } + + pub async fn get_branch_name(&self, id: &DocumentId) -> Option { + let Some(state) = self.inner.branch_db.get_branch_state(id).await else { + return None; + }; + Some(state.lock().await.name.clone()) + } + + pub async fn get_branch_state(&self, id: &DocumentId) -> Option { + let Some(state) = self.inner.branch_db.get_branch_state(id).await else { + return None; + }; + Some(state.lock().await.clone()) + } + + /// Returns the changes from the current branch. + pub async fn get_changes(&self) -> Vec { + let checked_out = self.inner.branch_db.get_checked_out_ref_mut().await; + let checked_out = checked_out.read().await; + let Some(checked_out) = checked_out.as_ref() else { + tracing::info!("Can't get changes; nothing checked out!"); + return Vec::new(); + }; + + let Some(branch_state) = self + .inner + .branch_db + .get_branch_state(&checked_out.branch) + .await + else { + tracing::info!("Can't get the checked out branch state; something must be wrong"); + return Vec::new(); + }; + + // TODO: we probably don't need to lock the branch state for this whole method + let branch_state = branch_state.lock().await; + let handle = branch_state.doc_handle.clone(); + let doc_id = handle.document_id(); + + let last_acked_heads = self + .inner + .peer_watcher + .get_server_info() + .await + .as_ref() + .and_then(|info| info.docs.get(&doc_id)) + .and_then(|state| state.last_acked_heads.clone()); + + let changes = handle.with_document(move |d| { + d.get_changes(&[]) + .to_vec() + .iter() + .map(|c| { + CommitInfo { + hash: c.hash(), + timestamp: c.timestamp(), + metadata: c + .message() + .and_then(|m| serde_json::from_str::(&m).ok()), + synced: false, // set later + summary: "".to_string(), // set later + } + }) + .collect::>() + }); + + // Check to see what the most recent synced commit is. + let mut synced_until_index = -1; + for (i, change) in changes.iter().enumerate() { + if last_acked_heads + .as_ref() + .is_some_and(|f| f.contains(&change.hash)) + { + synced_until_index = i as i32; + } + } + + changes + .into_iter() + .enumerate() + .map(|(i, change)| CommitInfo { + synced: (i as i32) <= synced_until_index, + ..change + }) + .collect() + } + + pub async fn get_checked_out_ref(&self) -> Option { + let checked_out_ref = self.inner.branch_db.get_checked_out_ref_mut().await; + return checked_out_ref.read().await.clone(); + } + + /// Sync the project state as best we can. + /// Make sure not to run two of these at once, for safety. + /// (It would likely be OK state-wise but weird results might happen on the UI side.) + /// Returns a vector of filesystem changes we performed. + pub async fn sync(&self) -> Vec { + // TODO (Lilith): There are inefficiencies with this strategy. + // Basically, every time we save a file, it'll do a bunch of extra work. + // It will first commit the changes, then it will check out the changes we just committed. + // No files will be written of course, but it will still walk the tree of changes. + // The same happens in reverse: when we check out a ref, it will attempt to commit and not + // find any actual changes. + // Maybe that's OK, we need to profile to see if it's a problem. + let changes = if Self::safe_to_update_godot() { + let goal_ref = { + let current_ref_lock = self.inner.branch_db.get_checked_out_ref_mut().await; + let current_ref_guard = current_ref_lock.read().await; + + let branch = match current_ref_guard.as_ref() { + Some(r) => r.branch.clone(), + None => { + // If we didn't find a current ref, nothing is checked out. + // Let's fix that by checking out the main branch. + let Some(main_branch) = self.inner.branch_db.get_main_branch().await else { + tracing::error!( + "No metadata doc checked out, or otherwise couldn't get main branch. Skipping sync!" + ); + return Vec::new(); + }; + main_branch + } + }; + let Some(goal_ref) = self.inner.branch_db.get_latest_ref_on_branch(&branch).await + else { + tracing::error!("Couldn't get the goal ref for branch {}", branch); + return Vec::new(); + }; + goal_ref + // guard dropped here + }; + // TODO (Lilith): minor problem, the checked out ref could change between these lines when the guard is dropped. + // That said, uhhhh I think that's fine? We're already syncing, so other sync methods shouldn't effect it. + // If we're doing some branch edits, like a merge or revert preview or something, that could mean we + // never checkout our desired ref... but we need to rethink this for branch swapping anyways. + // For branch swapping, we either need to checkout a ref elsewhere during a merge or something and prevent sync. + // Or, we could set a request_checkout tokio::Watch here on the driver that will check out a branch at the next opportunity. + self.inner.sync_automerge_to_fs.checkout_ref(goal_ref).await + } else { + Vec::new() + }; + + // Apply any watched FS updates to Automerge + self.inner.sync_fs_to_automerge.commit().await; + changes + } +} diff --git a/rust/src/project/fs_watcher.rs b/rust/src/project/fs_watcher.rs new file mode 100644 index 00000000..433921f5 --- /dev/null +++ b/rust/src/project/fs_watcher.rs @@ -0,0 +1,230 @@ +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + sync::Arc, + time::Duration, +}; + +use async_stream::stream; +use futures::Stream; +use md5::Digest; +use notify::{Config, RecommendedWatcher, RecursiveMode}; +use notify_debouncer_mini::{DebouncedEvent, new_debouncer_opt}; +use tokio::{ + sync::{ + Mutex, + mpsc::{self}, + }, + task::JoinSet, + time::sleep, +}; +use tokio_stream::wrappers::UnboundedReceiverStream; + +use crate::{ + fs::{ + file_utils::FileSystemEvent, + file_utils::{FileContent, calculate_file_hash, get_buffer_and_hash}, + }, + project::branch_db::BranchDb, +}; + +// TODO (Lilith): This works, but I'm not sure this complicated +// of a class is necessary... + +// Can we just do this naively, provide all FS events to the caller, +// then check the hashes against automerge to see if it's actually changed? + +/// Watches a directory for filesystem changes, and emits them as a stream. +#[derive(Debug, Clone)] +pub struct FileSystemWatcher { + watch_path: PathBuf, + file_hashes: Arc>>, + branch_db: BranchDb, + found_ignored_paths: Arc>>, +} + +impl FileSystemWatcher { + // We need to manually desugar the async function here because bug + // See: https://stackoverflow.com/questions/79851524 + fn initialize_file_hashes_recur( + &self, + watch_path: PathBuf, + ) -> impl Future> + Send { + async move { + let mut dir = tokio::fs::read_dir(watch_path).await?; + let mut set = JoinSet::new(); + while let Some(entry) = dir.next_entry().await? { + let path = entry.path(); + + // Skip if path matches any ignore pattern + if self.branch_db.should_ignore(&path) { + { + let mut found_ignored_paths = self.found_ignored_paths.lock().await; + found_ignored_paths.insert(path.clone()); + } + continue; + } + + if path.is_file() { + let file_hashes = self.file_hashes.clone(); + set.spawn(async move { + if let Some(hash) = calculate_file_hash(&path).await { + let mut file_hashes = file_hashes.lock().await; + file_hashes.insert(path, hash); + } + Ok(()) + }); + } else if path.is_dir() { + let path = path.clone(); + let this = self.clone(); + set.spawn(async move { this.initialize_file_hashes_recur(path).await }); + } + } + while let Some(_) = set.join_next().await {} + Ok(()) + } + } + + // Initialize the hash map with existing files + async fn initialize_file_hashes(&self) { + self.initialize_file_hashes_recur(self.watch_path.clone()) + .await; + } + + // Handle file creation and modification events + async fn handle_file_event( + &self, + path: PathBuf, + ) -> Result, notify::Error> { + // Skip if path matches any ignore pattern + if self.branch_db.should_ignore(&path) { + return Ok(None); + } + if !path.exists() { + // If the file doesn't exist, we want to emit a deleted event + let mut file_hashes = self.file_hashes.lock().await; + if file_hashes.contains_key(&path) { + file_hashes.remove(&path); + return Ok(Some(FileSystemEvent::FileDeleted(path))); + } + return Ok(None); + } + + if path.is_file() { + let mut result = get_buffer_and_hash(&path).await; + // TODO: is this still necessary? + if result.is_err() { + sleep(Duration::from_millis(100)).await; + result = get_buffer_and_hash(&path).await; + } + if result.is_err() { + tracing::error!("failed to get file content {:?}", result); + return Err(notify::Error::new(notify::ErrorKind::Generic( + "Failed to get file content".to_string(), + ))); + } + let (content, new_hash) = result.unwrap(); + let mut file_hashes = self.file_hashes.lock().await; + if file_hashes.contains_key(&path) { + let old_hash = file_hashes.get(&path).unwrap(); + if old_hash != &new_hash { + tracing::trace!( + "file {:?} changed, hash {:?} -> {:?}", + path, + old_hash, + new_hash + ); + file_hashes.insert(path.clone(), new_hash); + return Ok(Some(FileSystemEvent::FileModified( + path, + FileContent::from_buf(content), + ))); + } + } else { + // If the file is newly created, we want to emit a created event + tracing::trace!("file {:?} created, hash {:?}", path, new_hash); + file_hashes.insert(path.clone(), new_hash); + return Ok(Some(FileSystemEvent::FileCreated( + path, + FileContent::from_buf(content), + ))); + } + } + Ok(None) + } + + async fn process_notify_event(&self, event: DebouncedEvent) -> Option { + if self.branch_db.should_ignore(&event.path) { + return None; + } + let result = self.handle_file_event(event.path.clone()).await; + if let Ok(Some(ret)) = result { + return Some(ret); + } + return None; + } + + // Watch the filesystem for meaningful changes + pub async fn start_watching( + path: PathBuf, + branch_db: BranchDb, + ) -> impl Stream { + let (notify_tx, notify_rx) = mpsc::unbounded_channel(); + let notify_config = Config::default() + .with_follow_symlinks(false) + .with_ignore_globs( + branch_db + .get_ignore_globs() + .iter() + .map(|g| g.as_str().to_string()) + .collect(), + ); + + let debouncer_config = notify_debouncer_mini::Config::default() + .with_timeout(Duration::from_millis(100)) + .with_batch_mode(true) + .with_notify_config(notify_config); + + let mut debouncer = new_debouncer_opt::<_, RecommendedWatcher>( + debouncer_config, + move |event: Result, notify::Error>| { + notify_tx.send(event).unwrap(); + }, + ) + .unwrap(); + + // Begin the watch + // I'm assuming that notify uses good RAII and stops watching when we kill the handle.... hopefully. + debouncer + .watcher() + .watch(&path, RecursiveMode::Recursive) + .unwrap(); + + let this = FileSystemWatcher { + watch_path: path, + file_hashes: Arc::new(Mutex::new(HashMap::new())), + branch_db, + found_ignored_paths: Arc::new(Mutex::new(HashSet::new())), + }; + + this.initialize_file_hashes().await; + for path in this.found_ignored_paths.lock().await.iter() { + let _ret = debouncer.watcher().unwatch(path); + } + let stream = UnboundedReceiverStream::new(notify_rx); + // Process both file system events and update events + stream! { + // Handle file system events + for await notify_events in stream { + let Ok(notify_events) = notify_events else { + continue; + }; + for notify_event in notify_events { + if let Some(evt) = this.process_notify_event(notify_event).await { + yield evt; + } + } + } + } + } +} diff --git a/rust/src/project/new_project.rs b/rust/src/project/new_project.rs new file mode 100644 index 00000000..173c8031 --- /dev/null +++ b/rust/src/project/new_project.rs @@ -0,0 +1,294 @@ +use crate::diff::differ::ProjectDiff; +use crate::fs::file_utils::FileSystemEvent; +use crate::helpers::utils::{CommitInfo, summarize_changes}; +use crate::interop::godot_accessors:: + PatchworkConfigAccessor +; +use crate::project::branch_db::HistoryRef; +use crate::project::driver::Driver; +use crate::project::project_api::ChangeViewModel; +use automerge::ChangeHash; +use samod::DocumentId; +use std::path::PathBuf; +use std::time::SystemTime; +use std::cell::RefCell; +use std::{collections::HashMap, str::FromStr}; +use tokio::runtime::Runtime; + +/// Manages the state and operations of a Patchwork project within Godot. +/// Its API is exposed to GDScript via the GodotProject struct. +#[derive(Debug)] +pub struct Project { + // Project driver. If some, is running + pub(super) driver: Option, + project_dir: PathBuf, + pub(super) runtime: Runtime, + + // Tracked changes for the UI + pub(super) history: Vec, + pub(super) changes: HashMap, + last_ingest: (SystemTime, i32), + ingest_requested: bool, + last_known_branch: Option, + + // Cached diffs between refs + pub(super) diff_cache: RefCell>, +} + +/// The default server URL used for syncing Patchwork projects. Can be overridden by user or project configuration. +const DEFAULT_SERVER_URL: &str = "24.199.97.236:8085"; + +/// Notifications that can be emitted via process and consumed by GodotProject, in order to trigger signals to GDScript. +pub enum GodotProjectSignal { + CheckedOutBranch, + ChangesIngested, +} + +impl Project { + pub fn new(project_dir: PathBuf) -> Self { + // TODO (Lilith): ensure we make this work across the ENTIRE program, not just the driver. + // For now this encapsulates everything we multi-thread, since Project is the barrier for public async access. + // So it's fine. But if we want other code besides the driver to be multi-threaded... + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name("patchwork-driver-worker") + .build() + .unwrap(); + + Self { + driver: None, + project_dir, + runtime, + history: Vec::new(), + changes: HashMap::new(), + last_ingest: (SystemTime::UNIX_EPOCH, 0), + ingest_requested: true, + last_known_branch: None, + diff_cache: RefCell::new(HashMap::new()), + } + } + + /// Expensive operation to ingest all branch changes from automerge into the project data. + /// Should be called when we think there are new changes to process. + async fn ingest_changes(&mut self) { + let Some(driver) = self.driver.clone() else { + tracing::error!("Driver not started, can't ingest changes!"); + return; + }; + + tracing::info!("Ingesting changes..."); + + let changes = self.runtime.block_on(tokio::spawn(async move { + driver.get_changes().await + })).unwrap(); + + // Consume changes into self.changes + for change in changes { + self.history.push(change.hash); + self.changes.insert(change.hash, CommitInfo { + // the only actual things we're awaiting here are data locks + // so dw about parallelization + summary: self.get_change_summary(&change).await.unwrap_or("Invalid data".to_string()), + ..change + }); + } + } + + async fn get_change_summary(&self, change: &CommitInfo) -> Option { + let meta = change.metadata.as_ref(); + let author = meta?.username.as_ref()?; + + // merge commit + if let Some(merge_info) = &meta?.merge_metadata { + let merged_branch = &self + .driver + .as_ref() + .unwrap() // the driver had better be valid + .get_branch_name(&merge_info.merged_branch_id.clone()) + .await + .unwrap_or(merge_info.merged_branch_id.to_string()); + return Some(format!("↪ {author} merged {merged_branch}")); + } + + // revert commit + if let Some(revert_info) = &meta?.reverted_to { + let heads = revert_info + .iter() + .map(|s| &s[..7]) + .collect::>() + .join(", "); + return Some(format!("↩ {author} reverted to {heads}")); + } + + // initial commit + if change.is_setup() { + return Some(format!("Initialized repository")); + } + + return Some(summarize_changes(&author, meta?.changed_files.as_ref()?)); + } + + /// Request for a change ingestion to be dispatched. + fn request_ingestion(&mut self) { + self.ingest_requested = true; + } + + /// If able, ingest changes, clear the ingestion request, and return true. + /// Otherwise, return false. + fn try_ingest_changes(&mut self) -> bool { + // Do not try to ingest if we haven't requested. + if !self.ingest_requested { + return false; + } + let now = SystemTime::now(); + let Ok(last_diff) = now.duration_since(self.last_ingest.0) else { + return false; + }; + + // Impose an arbitrary cap on requests within a time period. + // This is so that immediate syncs -- such as those from a local server -- don't have to wait before getting synced. + // But it also prevents spam of like a hundred slowing down the ingestion. + if last_diff.as_millis() < 100 { + if self.last_ingest.1 >= 3 { + return false; + } + } else { + // since we're past the duration with no other requests, the counter resets. + self.last_ingest = (now, 0); + } + self.ingest_changes(); + self.ingest_requested = false; + self.last_ingest.1 += 1; + true + } + + pub fn get_cached_diff( + &self, + before: HistoryRef, + after: HistoryRef, + ) -> ProjectDiff { + self.diff_cache + .borrow_mut() + .entry((before.clone(), after.clone())) + .or_insert_with(|| self.get_diff(before, after)) + .clone() + } + + pub fn clear_diff_cache(&self) { + self.diff_cache.borrow_mut().clear(); + } + + pub fn get_diff( + &self, + before: HistoryRef, + after: HistoryRef, + ) -> ProjectDiff { + let Some(driver) = &self.driver else { + return ProjectDiff::default(); + }; + ProjectDiff::default() + + // TODO (Lilith): make this code work; ProjectDiff includes variants which can't be pushed across threads + // TODO: these can get expensive; propagate a spinner to the UI + // self.runtime.block_on(async move { + // tokio::spawn(async move { + // driver.get_diff(&before, &after).await + // }).await.unwrap() + // }) + } + + pub fn start(&mut self) { + if self.driver.is_some() { + tracing::error!("Driver is already started!"); + return; + } + + let storage_dir = self.project_dir.join(".patchwork"); + let server_url = { + let project = PatchworkConfigAccessor::get_project_value("server_url", ""); + let user = PatchworkConfigAccessor::get_user_value("server_url", ""); + if !project.is_empty() { + tracing::info!("Using project override for server url: {:?}", project); + project + } else if !user.is_empty() { + tracing::info!("Using user override for server url: {:?}", user); + user + } else { + let default = DEFAULT_SERVER_URL.to_string(); + tracing::info!("Using default server url: {:?}", default); + default + } + }; + + // If the metadata ID is not a valid document ID, give up. + // If it's an empty string, returns None so we can make a new doc. + let metadata_id = match Some(PatchworkConfigAccessor::get_project_value( + "project_doc_id", + "", + )) + .filter(|s| !s.is_empty()) + { + Some(s) => match DocumentId::from_str(&s) { + Ok(id) => Some(id), + Err(_) => { + tracing::error!("Invalid metadata document ID! Not starting driver."); + return; + } + }, + None => None, + }; + + // TODO (Lilith): Add support back for initially checking out a branch; probably once we're syncing + // let checked_out_branch_doc_id = PatchworkConfigAccessor::get_project_value("checked_out_branch_doc_id", ""); + tracing::info!( + "Starting GodotProject with metadata doc id: {:?}", + metadata_id + ); + + let project_dir = self.project_dir.clone(); + self.driver = self.runtime.block_on(async move { + // I think it's correct to spawn this on a different task explicitly, because block_on runs the future on the current thread, not a worker thread. + tokio::spawn(async move { + Driver::new(server_url, project_dir, storage_dir, metadata_id).await + }).await.unwrap() + }); + + if self.driver.is_none() { + tracing::error!("Could not start the driver!"); + return; + } + } + + pub fn stop(&mut self) { + self.driver.take(); + } + + pub fn process(&mut self, _delta: f64) -> (Vec, Vec) { + let Some(driver) = &self.driver else { + return Default::default(); + }; + let driver = driver.clone(); + + let mut signals: Vec = Vec::new(); + if self.try_ingest_changes() { + signals.push(GodotProjectSignal::ChangesIngested); + } + + // Run the main sync + let (changed_files, checked_out_ref) = self.runtime.block_on(async move { + tokio::spawn(async move { + (driver.sync().await, driver.get_checked_out_ref().await) + }).await.unwrap() + }); + + let current_branch = checked_out_ref.and_then(|r| Some(r.branch)); + if self.last_known_branch != current_branch { + signals.push(GodotProjectSignal::CheckedOutBranch); + } + + // TODO (Lilith): VERY IMPORTANT, set the patchwork config branch ID here!!! + // So that we save the branch ID for future checkouts. + + (changed_files, signals) + } +} diff --git a/rust/src/project/peer_watcher.rs b/rust/src/project/peer_watcher.rs new file mode 100644 index 00000000..cd0ef32c --- /dev/null +++ b/rust/src/project/peer_watcher.rs @@ -0,0 +1,88 @@ +use std::sync::Arc; + +use crate::{project::branch_db::BranchDb}; +use futures::StreamExt; +use samod::{Connection, ConnectionInfo, Repo}; +use tokio::{sync::Mutex, task::JoinHandle}; + +#[derive(Debug)] +pub struct PeerWatcher { + repo_handle: Repo, + branch_db: BranchDb, + server_info: Arc>>, + handle: Option>, +} + +impl Drop for PeerWatcher { + fn drop(&mut self) { + // Is this safe? Alternatively we could use a cancellation token + // I think it's safe though + self.handle.as_ref().map(|h| h.abort()); + } +} + +impl PeerWatcher { + pub fn new(repo_handle: Repo, branch_db: BranchDb) -> Self { + Self { + repo_handle, + branch_db, + server_info: Arc::new(Mutex::new(None)), + handle: None, + } + } + + pub async fn get_server_info(&self) -> Option { + return self.server_info.lock().await.clone(); + } + + pub fn start(&mut self) { + let repo_handle = self.repo_handle.clone(); + let server_info = self.server_info.clone(); + self.handle = Some(tokio::spawn(async move { + let (_, stream) = repo_handle.connected_peers(); + tokio::pin!(stream); + while let Some(peers) = stream.next().await { + // Currently, we only ever have 1 peer: the server. + // Therefore, this code expects that the server is the first and only peer, if it's connected. + // When we move to more peers, we'll need to figure out a way to identify the server here. + if let Some(info) = peers.first() { + Self::update_server_info(server_info.clone(), info.clone()).await; + } + } + })); + } + + async fn update_server_info( + old_info: Arc>>, + new_info: ConnectionInfo, + ) { + let mut server_info = old_info.lock().await; + if server_info.is_none() { + server_info.insert(new_info); + return; + } + let mut info = server_info.clone().unwrap(); + info.last_received = new_info.last_received; + info.last_sent = new_info.last_sent; + + for (doc_id, new_doc_state) in &new_info.docs { + if let Some(old_doc_state) = info.docs.get(doc_id) { + // If we got beheaded, skip this doc. + if new_doc_state + .last_acked_heads + .as_ref() + .is_some_and(|h| h.len() == 0) + && old_doc_state + .last_acked_heads + .as_ref() + .is_some_and(|h| h.len() > 0) + { + continue; + } + } + info.docs.insert(doc_id.clone(), new_doc_state.clone()); + } + + *server_info = Some(info); + } +} diff --git a/rust/src/project/project.rs b/rust/src/project/project.rs index e03d6e4f..a4101a14 100644 --- a/rust/src/project/project.rs +++ b/rust/src/project/project.rs @@ -1220,13 +1220,14 @@ impl Project { let Some(peer_connection_info) = peer_connection_info else { continue; }; - let _info = match self - .sync_server_connection_info - .as_mut() + let Some(current_info) = self.sync_server_connection_info else { + self.sync_server_connection_info = peer_connection_info.clone(); + continue; + }; + match self.sync_server_connection_info.as_mut() { None => { self.sync_server_connection_info = Some(peer_connection_info.clone()); - peer_connection_info } Some(sync_server_connection_info) => { @@ -1263,8 +1264,6 @@ impl Project { .docs .insert(doc_id.clone(), doc_state.clone()); }); - - peer_connection_info } }; self.request_ingestion(); diff --git a/rust/src/project/project_api_impl.rs b/rust/src/project/project_api_impl.rs index 115e6149..f0b37dbc 100644 --- a/rust/src/project/project_api_impl.rs +++ b/rust/src/project/project_api_impl.rs @@ -4,531 +4,676 @@ use automerge::ChangeHash; use samod::DocumentId; use tracing::instrument; -use crate::{diff::differ::ProjectDiff, helpers::utils::{BranchWrapper, CommitInfo, DiffWrapper, exact_human_readable_timestamp, human_readable_timestamp}, interop::godot_accessors::PatchworkConfigAccessor, project::{project::{CheckedOutBranchState, Project}, project_api::{BranchViewModel, ChangeViewModel, DiffViewModel, ProjectViewModel, SyncStatus}, project_driver::InputEvent}}; - +use crate::{ + diff::differ::ProjectDiff, + helpers::utils::{ + BranchWrapper, CommitInfo, DiffWrapper, exact_human_readable_timestamp, + human_readable_timestamp, + }, + interop::godot_accessors::PatchworkConfigAccessor, + project::{ + branch_db::HistoryRef, + new_project::Project, + project_api::{ + BranchViewModel, ChangeViewModel, DiffViewModel, ProjectViewModel, SyncStatus, + }, + }, +}; + +// TODO: Ideally this is actually a child of a new project submodule... +// that's so that it doesn't need pub(super) to acess private fields of +// itself. impl ProjectViewModel for Project { - fn has_project(&self) -> bool { - self.is_started() - } - - fn get_project_id(&self) -> Option { - self.get_project_doc_id() - } - - fn new_project(&mut self) { - if self.is_started() { - return; - } - self.start(); - } - - fn load_project(&mut self, id: &DocumentId) { - if self.is_started() { - return; - } - PatchworkConfigAccessor::set_project_value("project_doc_id", id.to_string().as_str()); - self.start(); - } - - fn clear_project(&mut self) { - if !self.is_started() { - return; - } + fn has_project(&self) -> bool { + self.driver.is_some() + } + + fn get_project_id(&self) -> Option { + let driver = self.driver.clone()?; + self.runtime + .block_on(tokio::spawn(async move { driver.get_metadata_doc().await })) + .unwrap() + } + + fn new_project(&mut self) { + if self.has_project() { + return; + } + self.start(); + } + + fn load_project(&mut self, id: &DocumentId) { + if self.has_project() { + return; + } + PatchworkConfigAccessor::set_project_value("project_doc_id", id.to_string().as_str()); + self.start(); + } + + fn clear_project(&mut self) { + if !self.has_project() { + return; + } self.stop(); - PatchworkConfigAccessor::set_user_value("user_name", ""); + PatchworkConfigAccessor::set_user_value("user_name", ""); PatchworkConfigAccessor::set_project_value("project_doc_id", ""); PatchworkConfigAccessor::set_project_value("checked_out_branch_doc_id", ""); - } + } - fn has_user_name(&self) -> bool { - PatchworkConfigAccessor::get_user_value("user_name", "") != "" - } + fn has_user_name(&self) -> bool { + PatchworkConfigAccessor::get_user_value("user_name", "") != "" + } - fn get_user_name(&self) -> String { - PatchworkConfigAccessor::get_user_value("user_name", "Anonymous") - } + fn get_user_name(&self) -> String { + PatchworkConfigAccessor::get_user_value("user_name", "Anonymous") + } fn set_user_name(&self, name: String) { - PatchworkConfigAccessor::set_user_value("user_name", &name); - if self.is_started() { - self.driver_input_tx - .unbounded_send(InputEvent::SetUserName { name }) - .unwrap(); - } - } - - fn can_create_merge_preview_branch(&self) -> bool { - match self.get_checked_out_branch_state() { - Some(branch_state) => { - !branch_state.is_main - }, - _ => false, - } - } - - fn create_merge_preview_branch(&mut self) { - let Some(checked_out_branch) = self.get_checked_out_branch_state() else { - return; - }; - let Some(fork_info) = &checked_out_branch.fork_info else { - return; - }; - self.create_merge_preview_branch_between( - checked_out_branch.doc_handle.document_id().clone(), - fork_info.forked_from.clone()); - } + PatchworkConfigAccessor::set_user_value("user_name", &name); + let Some(driver) = self.driver.clone() else { + return; + }; + self.runtime.block_on(tokio::spawn(async move { + driver.set_username(Some(name)).await + })); + } + + fn can_create_merge_preview_branch(&self) -> bool { + // TODO (Lilith): implement + return false; + // match self.get_checked_out_branch_state() { + // Some(branch_state) => !branch_state.is_main, + // _ => false, + // } + } + + fn create_merge_preview_branch(&mut self) { + // TODO (Lilith): implement + return; + // let Some(checked_out_branch) = self.get_checked_out_branch_state() else { + // return; + // }; + // let Some(fork_info) = &checked_out_branch.fork_info else { + // return; + // }; + // self.create_merge_preview_branch_between( + // checked_out_branch.doc_handle.document_id().clone(), + // fork_info.forked_from.clone(), + // ); + } fn can_create_revert_preview_branch(&self, head: ChangeHash) -> bool { - if self.is_revert_preview_branch_active() || self.is_merge_preview_branch_active() { return false; } - if self.get_change(head).is_some_and(|c| - // Allow reverts for only the second setup commit - !c.is_setup() || self.get_branch_history().iter().position(|&h| h == head) == Some(1)) { - return self.get_checked_out_branch_state().is_some(); - } - false - } + // TODO (Lilith): implement + return false; + // if self.is_revert_preview_branch_active() || self.is_merge_preview_branch_active() { + // return false; + // } + // if self.get_change(head).is_some_and(|c| + // // Allow reverts for only the second setup commit + // !c.is_setup() || self.get_branch_history().iter().position(|&h| h == head) == Some(1)) + // { + // return self.get_checked_out_branch_state().is_some(); + // } + // false + } fn create_revert_preview_branch(&mut self, head: ChangeHash) { - let Some(checked_out_branch) = self.get_checked_out_branch_state() else { - return; - }; - self.create_revert_preview_branch_for( - checked_out_branch.doc_handle.document_id().clone(), - vec![head]); - } - - fn is_revert_preview_branch_active(&self) -> bool { - let branch_state = self.get_checked_out_branch_state(); - match branch_state { - Some(state) => - state.revert_info.is_some(), - _ => false - } - } - - fn is_merge_preview_branch_active(&self) -> bool { - let branch_state = self.get_checked_out_branch_state(); - match branch_state { - Some(state) => - state.merge_info.is_some(), - _ => false - } - } - - fn is_safe_to_merge(&self) -> bool { - let Some(current_branch) = self.get_checked_out_branch_state() else { - return false; - }; - let Some(merge_info) = current_branch.merge_info.as_ref() else { - return false; - }; - let Some(fork_info) = current_branch.fork_info.as_ref() else { - return false; - }; - - let source_branch = self.branch_states.get(&fork_info.forked_from); - let dest_branch = self.branch_states.get(&merge_info.merge_into); - - let Some(dest_branch) = dest_branch else { - return false; - }; - - source_branch.is_some_and(|s| - s.fork_info.as_ref().is_some_and(|i| - i.forked_at == dest_branch.synced_heads)) - } + // TODO (Lilith): implement + return; + // let Some(checked_out_branch) = self.get_checked_out_branch_state() else { + // return; + // }; + // self.create_revert_preview_branch_for( + // checked_out_branch.doc_handle.document_id().clone(), + // vec![head], + // ); + } + + fn is_revert_preview_branch_active(&self) -> bool { + // TODO (Lilith): implement + return false; + // let branch_state = self.get_checked_out_branch_state(); + // match branch_state { + // Some(state) => state.revert_info.is_some(), + // _ => false, + // } + } + + fn is_merge_preview_branch_active(&self) -> bool { + // TODO (Lilith): implement + return false; + // let branch_state = self.get_checked_out_branch_state(); + // match branch_state { + // Some(state) => state.merge_info.is_some(), + // _ => false, + // } + } + + fn is_safe_to_merge(&self) -> bool { + // TODO (Lilith): implement + return false; + // let Some(current_branch) = self.get_checked_out_branch_state() else { + // return false; + // }; + // let Some(merge_info) = current_branch.merge_info.as_ref() else { + // return false; + // }; + // let Some(fork_info) = current_branch.fork_info.as_ref() else { + // return false; + // }; + + // let source_branch = self.branch_states.get(&fork_info.forked_from); + // let dest_branch = self.branch_states.get(&merge_info.merge_into); + + // let Some(dest_branch) = dest_branch else { + // return false; + // }; + + // source_branch.is_some_and(|s| { + // s.fork_info + // .as_ref() + // .is_some_and(|i| i.forked_at == dest_branch.synced_heads) + // }) + } fn confirm_preview_branch(&mut self) { - let Some(branch_state) = self.get_checked_out_branch_state().cloned() else { - return; - }; - let Some(fork_info) = &branch_state.fork_info else { - return; - }; - if let Some(revert_info) = &branch_state.revert_info { - self.delete_branch(branch_state.doc_handle.document_id().clone()); - self.checkout_branch(fork_info.forked_from.clone()); - self.revert_to_heads(revert_info.reverted_to.clone()); - } - else if let Some(merge_info) = branch_state.merge_info { - self.merge_branch(branch_state.doc_handle.document_id().clone(), merge_info.merge_into) - } - } + // TODO (Lilith): implement + return; + // let Some(branch_state) = self.get_checked_out_branch_state().cloned() else { + // return; + // }; + // let Some(fork_info) = &branch_state.fork_info else { + // return; + // }; + // if let Some(revert_info) = &branch_state.revert_info { + // self.delete_branch(branch_state.doc_handle.document_id().clone()); + // self.checkout_branch(fork_info.forked_from.clone()); + // self.revert_to_heads(revert_info.reverted_to.clone()); + // } else if let Some(merge_info) = branch_state.merge_info { + // self.merge_branch( + // branch_state.doc_handle.document_id().clone(), + // merge_info.merge_into, + // ) + // } + } fn discard_preview_branch(&mut self) { - let Some(branch_state) = self.get_checked_out_branch_state().cloned() else { - return; - }; - let Some(fork_info) = &branch_state.fork_info else { - return; - }; - self.delete_branch(branch_state.doc_handle.document_id().clone()); - self.checkout_branch(fork_info.forked_from.clone()); - } - - fn get_branch_history(&self) -> Vec { - let Some(branch_state) = self.get_checked_out_branch_state().cloned() else { - return vec!(); - }; - self.history.iter() - .filter(|item| - self.changes.get(item).is_some_and(|i| - i.metadata.as_ref().is_some_and(|m| - m.branch_id.as_ref().is_some_and(|id| - id == branch_state.doc_handle.document_id())))) - .map(|hash| hash.clone()) - .collect::>() - } + // TODO (Lilith): implement + return; + // let Some(branch_state) = self.get_checked_out_branch_state().cloned() else { + // return; + // }; + // let Some(fork_info) = &branch_state.fork_info else { + // return; + // }; + // self.delete_branch(branch_state.doc_handle.document_id().clone()); + // self.checkout_branch(fork_info.forked_from.clone()); + } + + fn get_branch_history(&self) -> Vec { + self.history.clone() + } fn get_sync_status(&self) -> SyncStatus { - if !self.is_started() { - // We have no reason to be connected, therefore just mark it as OK. - return SyncStatus::UpToDate; - } - let Some(info) = &self.sync_server_connection_info else { - return SyncStatus::Unknown; - }; - let Some(branch) = self.get_checked_out_branch_state() else { - return SyncStatus::Unknown; - }; - let Some(status) = info.docs.get(&branch.doc_handle.document_id()) else { - return SyncStatus::Unknown; - }; - let is_connected = info.last_received.is_some(); - if status.last_acked_heads.as_ref().is_some_and(|s| s == &branch.synced_heads) { - if is_connected { - return SyncStatus::UpToDate; - } - return SyncStatus::Disconnected(0); - } + // TODO (Lilith): implement + return SyncStatus::Unknown; + // if !self.has_project() { + // // We have no reason to be connected, therefore just mark it as OK. + // return SyncStatus::UpToDate; + // } - if is_connected { - return SyncStatus::Syncing; - } + // let Some(info) = &self.sync_server_connection_info else { + // return SyncStatus::Unknown; + // }; + // let Some(branch) = self.get_checked_out_branch_state() else { + // return SyncStatus::Unknown; + // }; + // let Some(status) = info.docs.get(&branch.doc_handle.document_id()) else { + // return SyncStatus::Unknown; + // }; + // let is_connected = info.last_received.is_some(); + // if status + // .last_acked_heads + // .as_ref() + // .is_some_and(|s| s == &branch.synced_heads) + // { + // if is_connected { + // return SyncStatus::UpToDate; + // } + // return SyncStatus::Disconnected(0); + // } + + // if is_connected { + // return SyncStatus::Syncing; + // } - let unsynced_count = self.changes - .iter() - .filter(|(_hash, c)|!c.synced) - .count(); + // let unsynced_count = self.changes.iter().filter(|(_hash, c)| !c.synced).count(); - return SyncStatus::Disconnected(unsynced_count) + // return SyncStatus::Disconnected(unsynced_count); } fn print_sync_debug(&self) { - if !self.is_started() { - return; - } - let Some(info) = &self.sync_server_connection_info else { - tracing::debug!("Sync info UNAVAILABLE!!!"); - return; + // TODO (Lilith): implement + return; + // if !self.is_started() { + // return; + // } + // let Some(info) = &self.sync_server_connection_info else { + // tracing::debug!("Sync info UNAVAILABLE!!!"); + // return; + // }; + // let is_connected = info.last_received.is_some(); + + // // fn time(t: Option) -> String { + // // let Some(t) = t else { + // // return "-".to_string() + // // }; + // // human_readable_timestamp(t + // // .duration_since(UNIX_EPOCH) + // // .unwrap() + // // .as_millis() + // // .try_into().unwrap()) + // // } + + // tracing::debug!("Sync info ==========================="); + // tracing::debug!("is connected: {is_connected}"); + // tracing::debug!("last received: {:?}", info.last_received); + // tracing::debug!("last sent: {:?}", info.last_sent); + + // if let Some(branch) = self.get_checked_out_branch_state() { + // if let Some(status) = info.docs.get(&branch.doc_handle.document_id()) { + // tracing::debug!("\t{}:", branch.name); + // tracing::debug!("\tacked heads: {:?}", status.last_acked_heads); + // tracing::debug!("\tsent heads: {:?}", status.last_sent_heads); + // tracing::debug!("\tlast sent: {:?}", status.last_sent); + // tracing::debug!("\tlast sent: {:?}", status.last_received); + // } + // } + // tracing::debug!("====================================="); + } + + fn get_branch(&self, id: &DocumentId) -> Option> { + let driver = self.driver.clone()?; + let id = id.clone(); + let state = self + .runtime + .block_on(tokio::spawn( + async move { driver.get_branch_state(&id).await }, + )) + .unwrap(); + let Some(state) = state else { + return None; }; - let is_connected = info.last_received.is_some(); - // fn time(t: Option) -> String { - // let Some(t) = t else { - // return "-".to_string() + // TODO (Lilith): implement + // let mut children = self + // .branch_states + // .values() + // .filter(|b| { + // b.fork_info + // .as_ref() + // .is_some_and(|i| i.forked_from == id.clone()) + // }) + // .map(|b| b.doc_handle.document_id().clone()) + // .collect::>(); + + // children.sort_by(|a, b| { + // let a_state = self.branch_states.get(&a); + // let b_state = self.branch_states.get(&b); + // let Some(a_state) = a_state else { + // return std::cmp::Ordering::Less; + // }; + // let Some(b_state) = b_state else { + // return std::cmp::Ordering::Greater; // }; - // human_readable_timestamp(t - // .duration_since(UNIX_EPOCH) - // .unwrap() - // .as_millis() - // .try_into().unwrap()) + // a_state + // .name + // .to_lowercase() + // .cmp(&b_state.name.to_lowercase()) + // }); + + Some(BranchWrapper { + state: state.clone(), + // children, + children: Vec::new(), + }) + } + + fn get_main_branch(&self) -> Option { + let driver = self.driver.clone()?; + let id = self + .runtime + .block_on(tokio::spawn(async move { driver.get_main_branch().await })) + .unwrap()?; + self.get_branch(&id) + } + + fn get_checked_out_branch(&self) -> Option { + let driver = self.driver.clone()?; + let id = self + .runtime + .block_on(tokio::spawn( + async move { driver.get_checked_out_ref().await }, + )) + .unwrap()?; + self.get_branch(&id.branch) + } + + #[instrument(skip(self), fields(name = ?name), level = tracing::Level::INFO)] + fn create_branch(&mut self, name: String) { + // TODO (Lilith): implement + return; + // println!(""); + // tracing::info!("******** CREATE BRANCH"); + // println!(""); + // let source_branch_doc_id = match &self.get_checked_out_branch_state() { + // Some(branch_state) => branch_state.doc_handle.document_id(), + // None => { + // panic!("couldn't create branch, no checked out branch"); + // } + // }; + + // self.driver_input_tx + // .unbounded_send(InputEvent::CreateBranch { + // name, + // source_branch_doc_id: source_branch_doc_id.clone(), + // }) + // .unwrap(); + + // // TODO: do we want to set this? or let _process set it? + // self.checked_out_branch_state = + // CheckedOutBranchState::NothingCheckedOut(Some(source_branch_doc_id.clone())); + // // self.checked_out_branch_state = CheckedOutBranchState::NothingCheckedOut(None); + } + + fn checkout_branch(&mut self, branch_doc_id: DocumentId) { + // TODO (Lilith): implement + return; + // let current_branch = match &self.checked_out_branch_state { + // CheckedOutBranchState::CheckedOut(doc_id, _) => Some(doc_id.clone()), + // CheckedOutBranchState::CheckingOut(doc_id, _) => { + // tracing::error!( + // "**@#%@#%!@#%#@!*** CHECKING OUT BRANCH WHILE STILL CHECKING OUT?!?!?! {:?}", + // doc_id + // ); + // Some(doc_id.clone()) + // } + // CheckedOutBranchState::NothingCheckedOut(current_branch_id) => { + // tracing::warn!("Checking out a branch while not checked out on any branch????"); + // current_branch_id.clone() + // } + // }; + // let target_branch_state = match self.branch_states.get(&branch_doc_id) { + // Some(branch_state) => branch_state, + // None => panic!("couldn't checkout branch, branch doc id not found"), + // }; + // println!(""); + // tracing::debug!("******** CHECKOUT: {:?}\n", target_branch_state.name); + // println!(""); + + // if target_branch_state.synced_heads + // == target_branch_state + // .doc_handle + // .with_document(|d| d.get_heads()) + // { + // self.checked_out_branch_state = + // CheckedOutBranchState::CheckedOut(branch_doc_id.clone(), current_branch.clone()); + // self.just_checked_out_new_branch = true; + // } else { + // tracing::debug!( + // "checked out branch {:?} has unsynced heads", + // target_branch_state.name + // ); + // self.checked_out_branch_state = + // CheckedOutBranchState::CheckingOut(branch_doc_id.clone(), current_branch.clone()); // } + } - tracing::debug!("Sync info ==========================="); - tracing::debug!("is connected: {is_connected}"); - tracing::debug!("last received: {:?}", info.last_received); - tracing::debug!("last sent: {:?}", info.last_sent); - - if let Some(branch) = self.get_checked_out_branch_state() { - if let Some(status) = info.docs.get(&branch.doc_handle.document_id()) { - tracing::debug!("\t{}:", branch.name); - tracing::debug!("\tacked heads: {:?}", status.last_acked_heads); - tracing::debug!("\tsent heads: {:?}", status.last_sent_heads); - tracing::debug!("\tlast sent: {:?}", status.last_sent); - tracing::debug!("\tlast sent: {:?}", status.last_received); - } - } - tracing::debug!("====================================="); + fn get_change(&self, hash: ChangeHash) -> Option<&impl ChangeViewModel> { + self.changes.get(&hash) } - fn get_branch(&self, id: &DocumentId) -> Option> { - let state = self - .branch_states - .get(&id)?; - - let mut children = self.branch_states - .values() - .filter(|b| - b.fork_info.as_ref().is_some_and(|i| - i.forked_from == id.clone())) - .map(|b| b.doc_handle.document_id().clone()) - .collect::>(); - - children.sort_by(|a, b| { - let a_state = self.branch_states.get(&a); - let b_state = self.branch_states.get(&b); - let Some(a_state) = a_state else { - return std::cmp::Ordering::Less; - }; - let Some(b_state) = b_state else { - return std::cmp::Ordering::Greater; - }; - a_state.name.to_lowercase().cmp(&b_state.name.to_lowercase()) - }); - - Some(BranchWrapper { - state: state.clone(), - children - }) - } - - fn get_main_branch(&self) -> Option { - let state = self - .branch_states - .values() - .find(|branch_state| branch_state.is_main); - self.get_branch(&state?.doc_handle.document_id()) - } - - fn get_checked_out_branch(&self) -> Option { - let state = self.get_checked_out_branch_state(); - self.get_branch(&state?.doc_handle.document_id()) - } - - #[instrument(skip(self), fields(name = ?name), level = tracing::Level::INFO)] - fn create_branch(&mut self, name: String) { - println!(""); - tracing::info!("******** CREATE BRANCH"); - println!(""); - let source_branch_doc_id = match &self.get_checked_out_branch_state() { - Some(branch_state) => branch_state.doc_handle.document_id(), - None => { - panic!("couldn't create branch, no checked out branch"); - } - }; + fn get_default_diff(&self) -> Option { + let heads_before; + let heads_after; + let driver = self.driver.clone()?; + let branch_state = self + .runtime + .block_on(tokio::spawn(async move { + driver + .get_branch_state(&driver.get_checked_out_ref().await?.branch) + .await + })) + .unwrap()?; + + // There is no default diff for the main branch! + if branch_state.is_main { + return None; + } - self.driver_input_tx - .unbounded_send(InputEvent::CreateBranch { - name, - source_branch_doc_id: source_branch_doc_id.clone(), - }) - .unwrap(); + if self.is_merge_preview_branch_active() { + heads_before = branch_state.merge_info.as_ref()?.merge_at.clone(); + } + // revert preview and regular branch both use forked_at + else { + heads_before = branch_state.fork_info.as_ref()?.forked_at.clone(); + } - // TODO: do we want to set this? or let _process set it? - self.checked_out_branch_state = CheckedOutBranchState::NothingCheckedOut(Some(source_branch_doc_id.clone())); - // self.checked_out_branch_state = CheckedOutBranchState::NothingCheckedOut(None); - } - - fn checkout_branch(&mut self, branch_doc_id: DocumentId) { - let current_branch = match &self.checked_out_branch_state { - CheckedOutBranchState::CheckedOut(doc_id, _) => Some(doc_id.clone()), - CheckedOutBranchState::CheckingOut(doc_id, _) => { - tracing::error!("**@#%@#%!@#%#@!*** CHECKING OUT BRANCH WHILE STILL CHECKING OUT?!?!?! {:?}", doc_id); - Some(doc_id.clone()) - }, - CheckedOutBranchState::NothingCheckedOut(current_branch_id) => { - tracing::warn!("Checking out a branch while not checked out on any branch????"); - current_branch_id.clone() - } - }; - let target_branch_state = match self.branch_states.get(&branch_doc_id) { - Some(branch_state) => branch_state, - None => panic!("couldn't checkout branch, branch doc id not found") - }; - println!(""); - tracing::debug!("******** CHECKOUT: {:?}\n", target_branch_state.name); - println!(""); - - if target_branch_state.synced_heads == target_branch_state.doc_handle.with_document(|d| d.get_heads()) { - self.checked_out_branch_state = - CheckedOutBranchState::CheckedOut( - branch_doc_id.clone(), - current_branch.clone()); - self.just_checked_out_new_branch = true; + // TODO (Lilith): Make synced heads work again + heads_after = branch_state.synced_heads.clone(); + + // generate the summary + let title; + if self.is_merge_preview_branch_active() { + let source_name = self + .get_branch(&branch_state.fork_info.as_ref()?.forked_from.clone())? + .get_name(); + let target_name = self + .get_branch(&branch_state.merge_info.as_ref()?.merge_into.clone())? + .get_name(); + title = format!("Showing changes for {} -> {}", source_name, target_name); + } else if self.is_revert_preview_branch_active() { + let source_name = self + .get_branch(&branch_state.fork_info.as_ref()?.forked_from.clone())? + .get_name(); + // assume reverted_to is always just 1 hash + let short_heads = &branch_state + .revert_info + .as_ref()? + .reverted_to + .first()? + .to_string()[..7]; + title = format!( + "Showing changes for {} reverted to {}", + source_name, short_heads + ); } else { - tracing::debug!("checked out branch {:?} has unsynced heads", target_branch_state.name); - self.checked_out_branch_state = - CheckedOutBranchState::CheckingOut( - branch_doc_id.clone(), - current_branch.clone() - ); + let source_name = self + .get_branch(&branch_state.fork_info.as_ref()?.forked_from.clone())? + .get_name(); + title = format!( + "Showing changes from {} -> {}", + source_name, branch_state.name + ); } + + let before = HistoryRef { + branch: branch_state.doc_handle.document_id().clone(), + heads: heads_before, + }; + + let after = HistoryRef { + branch: branch_state.doc_handle.document_id().clone(), + heads: heads_after, + }; + + Some(DiffWrapper { + diff: self.get_cached_diff(before, after), + title, + }) } - fn get_change(&self, hash: ChangeHash) -> Option<&impl ChangeViewModel> { - self.changes.get(&hash) - } - - fn get_default_diff(&self) -> Option { - let heads_before; - let heads_after; - let branch_state = self.get_checked_out_branch_state()?; - - // There is no default diff for the main branch! - if branch_state.is_main { - return None; - } - - if self.is_merge_preview_branch_active() { - heads_before = branch_state.merge_info.as_ref()?.merge_at.clone(); - } - // revert preview and regular branch both use forked_at - else { - heads_before = branch_state.fork_info.as_ref()?.forked_at.clone(); - } - - heads_after = branch_state.synced_heads.clone(); - - // generate the summary - let title; - if self.is_merge_preview_branch_active() { - let source_name = self.get_branch(&branch_state.fork_info.as_ref()?.forked_from.clone())?.get_name(); - let target_name = self.get_branch(&branch_state.merge_info.as_ref()?.merge_into.clone())?.get_name(); - title = format!("Showing changes for {} -> {}", source_name, target_name); - } - else if self.is_revert_preview_branch_active() { - let source_name = self.get_branch(&branch_state.fork_info.as_ref()?.forked_from.clone())?.get_name(); - // assume reverted_to is always just 1 hash - let short_heads = &branch_state.revert_info.as_ref()?.reverted_to - .first()?.to_string()[..7]; - title = format!("Showing changes for {} reverted to {}", source_name, short_heads); - } - else { - let source_name = self.get_branch(&branch_state.fork_info.as_ref()?.forked_from.clone())?.get_name(); - title = format!("Showing changes from {} -> {}", source_name, branch_state.name); - } - - Some(DiffWrapper { - diff: self.get_cached_diff(heads_before, heads_after), - title - }) - } - - fn get_diff(&self, selected_hash: ChangeHash) -> Option { - let change = self.changes.get(&selected_hash)?; - if change.is_setup() { - return None; - } - let heads_before; - let heads_after = vec!(change.hash); - - let history = self.get_branch_history(); - let mut prev_hash = None; - for (i, el) in history.iter().enumerate() { - if *el == selected_hash { - prev_hash = history.get(i - 1).copied(); - break; - } - } - - if let Some(prev_hash) = prev_hash { - heads_before = vec!(prev_hash); - } - else { - heads_before = self.get_checked_out_branch_state()?.fork_info.as_ref()?.forked_at.clone(); - } - - Some(DiffWrapper { - diff: self.get_cached_diff(heads_before, heads_after), - title: format!("Showing changes from {} - {}", - change.get_summary(), change.get_human_timestamp()) - }) - } + fn get_diff(&self, selected_hash: ChangeHash) -> Option { + let change = self.changes.get(&selected_hash)?; + if change.is_setup() { + return None; + } + let heads_before; + let heads_after = vec![change.hash]; + + let history = self.get_branch_history(); + let mut prev_hash = None; + for (i, el) in history.iter().enumerate() { + if *el == selected_hash { + prev_hash = history.get(i - 1).copied(); + break; + } + } + + let driver = self.driver.clone()?; + let branch_state = self + .runtime + .block_on(tokio::spawn(async move { + driver + .get_branch_state(&driver.get_checked_out_ref().await?.branch) + .await + })) + .unwrap()?; + + if let Some(prev_hash) = prev_hash { + heads_before = vec![prev_hash]; + } else { + heads_before = branch_state.fork_info.as_ref()?.forked_at.clone(); + } + + let before = HistoryRef { + branch: branch_state.doc_handle.document_id().clone(), + heads: heads_before, + }; + + let after = HistoryRef { + branch: branch_state.doc_handle.document_id().clone(), + heads: heads_after, + }; + + Some(DiffWrapper { + diff: self.get_cached_diff(before, after), + title: format!( + "Showing changes from {} - {}", + change.get_summary(), + change.get_human_timestamp() + ), + }) + } } impl ChangeViewModel for CommitInfo { - fn get_hash(&self) -> ChangeHash { - self.hash - } - - fn get_username(&self) -> String { - if let Some(meta) = &self.metadata { - if let Some(author) = &meta.username { - return author.clone(); - } - }; - "Anonymous".to_string() - } - - fn is_synced(&self) -> bool { - self.synced - } - - fn get_summary(&self) -> String { - self.summary.clone() - } - - fn is_merge(&self) -> bool { + fn get_hash(&self) -> ChangeHash { + self.hash + } + + fn get_username(&self) -> String { + if let Some(meta) = &self.metadata { + if let Some(author) = &meta.username { + return author.clone(); + } + }; + "Anonymous".to_string() + } + + fn is_synced(&self) -> bool { + self.synced + } + + fn get_summary(&self) -> String { + self.summary.clone() + } + + fn is_merge(&self) -> bool { let Some(meta) = &self.metadata else { return false; }; return meta.merge_metadata.is_some(); - } + } - fn is_setup(&self) -> bool { + fn is_setup(&self) -> bool { let Some(meta) = &self.metadata else { return false; }; return meta.is_setup.unwrap_or(false); - } + } - fn get_exact_timestamp(&self) -> String { + fn get_exact_timestamp(&self) -> String { exact_human_readable_timestamp(self.timestamp) - } + } - fn get_human_timestamp(&self) -> String { + fn get_human_timestamp(&self) -> String { human_readable_timestamp(self.timestamp) - } + } - fn get_merge_id(&self) -> Option { - Some(self.metadata.as_ref()? - .merge_metadata.as_ref()? - .merged_branch_id.clone()) - } + fn get_merge_id(&self) -> Option { + Some( + self.metadata + .as_ref()? + .merge_metadata + .as_ref()? + .merged_branch_id + .clone(), + ) + } } impl BranchViewModel for BranchWrapper { - fn get_id(&self) -> DocumentId { - self.state.doc_handle.document_id().clone() - } + fn get_id(&self) -> DocumentId { + self.state.doc_handle.document_id().clone() + } - fn get_name(&self) -> String { - self.state.name.clone() - } + fn get_name(&self) -> String { + self.state.name.clone() + } - fn get_parent(&self) -> Option { - Some(self.state.fork_info.as_ref()?.forked_from.clone()) - } + fn get_parent(&self) -> Option { + Some(self.state.fork_info.as_ref()?.forked_from.clone()) + } - fn get_children(&self) -> Vec { - self.children.clone() - } + fn get_children(&self) -> Vec { + self.children.clone() + } - fn is_available(&self) -> bool { - !self.get_merge_into().is_some() && !self.get_reverted_to().is_some() - } + fn is_available(&self) -> bool { + !self.get_merge_into().is_some() && !self.get_reverted_to().is_some() + } - fn is_loaded(&self) -> bool { + fn is_loaded(&self) -> bool { // we shouldn't have branches that don't have any changes but sometimes // the branch docs are not synced correctly so this flag is used in the UI to // indicate that the branch is not loaded and prevent users from checking it out - !self.state.doc_handle.with_document(|d| d.get_heads().len() == 0) - } + !self + .state + .doc_handle + .with_document(|d| d.get_heads().len() == 0) + } - fn get_reverted_to(&self) -> Option { - Some(self.state.revert_info.as_ref()?.reverted_to.first()?.clone()) - } + fn get_reverted_to(&self) -> Option { + Some( + self.state + .revert_info + .as_ref()? + .reverted_to + .first()? + .clone(), + ) + } - fn get_merge_into(&self) -> Option { - Some(self.state.merge_info.as_ref()?.merge_into.clone()) - } + fn get_merge_into(&self) -> Option { + Some(self.state.merge_info.as_ref()?.merge_into.clone()) + } } impl DiffViewModel for DiffWrapper { - fn get_diff(&self) -> &ProjectDiff { - &self.diff - } + fn get_diff(&self) -> &ProjectDiff { + &self.diff + } - fn get_title(&self) -> &String { - &self.title - } + fn get_title(&self) -> &String { + &self.title + } } diff --git a/rust/src/project/project_driver.rs b/rust/src/project/project_driver.rs index 957df102..4c935003 100644 --- a/rust/src/project/project_driver.rs +++ b/rust/src/project/project_driver.rs @@ -163,25 +163,18 @@ struct DriverState { heads_in_frontend: HashMap> } -pub enum ConnectionThreadError { - ConnectionThreadDied(String), - ConnectionThreadError(String), -} #[derive(Debug)] pub struct ProjectDriver { - runtime: Runtime, - repo_handle: Repo, - server_url: String, - connection_thread_output_rx: Option>, - retries: u32, - connection_thread: Option>, - spawned_thread: Option>, - server_connected: Arc + // todo: make these not pub + pub runtime: Runtime, + pub repo_handle: Repo, + pub spawned_thread: Option>, } impl ProjectDriver { pub async fn create(storage_folder_path: String, server_url: String) -> Self { + // TODO: ensure we make this work across the program. Initialize it only once, etc. let runtime: Runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) .enable_all() @@ -207,12 +200,7 @@ impl ProjectDriver { return Self { runtime, repo_handle, - server_url, - retries: 0, - connection_thread_output_rx: None, - connection_thread: None, - spawned_thread: None, - server_connected: Arc::new(AtomicBool::new(false)) + spawned_thread: None }; } @@ -223,136 +211,17 @@ impl ProjectDriver { branches_metadata_doc_id: Option, user_name: Option, ) { - if self.connection_thread.is_some() || self.spawned_thread.is_some() { - tracing::warn!("driver already spawned"); - return; - } - - self.respawn_connection_thread(); - // Spawn sync task for all doc handles self.spawned_thread = Some(self.spawn_driver_task(rx, tx, branches_metadata_doc_id, &user_name)); } pub fn teardown(&mut self) { - if let Some(connection_thread) = self.connection_thread.take() { - connection_thread.abort(); - } if let Some(spawned_thread) = self.spawned_thread.take() { spawned_thread.abort(); } } - fn connection_thread_died(&self) -> bool { - if let Some(connection_thread) = &self.connection_thread { - return connection_thread.is_finished(); - } - false - } - - pub fn connection_thread_get_last_error(&mut self) -> Option { - if let Some(connection_thread_rx) = &mut self.connection_thread_output_rx { - let mut error_str = String::new(); - while let Ok(Some(error)) = connection_thread_rx.try_next() { - error_str.push_str("\n"); - error_str.push_str(&error); - } - if self.connection_thread_died() { - self.retries += 1; - return Some(ConnectionThreadError::ConnectionThreadDied(error_str)); - } - if !error_str.is_empty() { - return Some(ConnectionThreadError::ConnectionThreadError(error_str)); - } - } - None - } - - pub fn respawn_connection_thread(&mut self) -> bool { - if let Some(connection_thread) = self.connection_thread.take() { - if !connection_thread.is_finished() { - tracing::warn!("WARNING: connection thread is not finished, aborting"); - connection_thread.abort(); - } - } - if self.retries > 6 { - tracing::error!("connection thread failed too many times, aborting"); - return false; - } - let (connection_thread_tx, connection_thread_rx) = futures::channel::mpsc::unbounded(); - self.connection_thread_output_rx = Some(connection_thread_rx); - self.connection_thread = Some(self.spawn_connection_task(connection_thread_tx)); - return true; - } - - fn spawn_connection_task(&self, connection_thread_tx: UnboundedSender) -> JoinHandle<()> { - let repo_handle_clone = self.repo_handle.clone(); - let server_connected = self.server_connected.clone(); - let retries = self.retries; - let server_url = self.server_url.clone(); - return self.runtime.spawn(async move { - tracing::info!("start a client"); - let backoff = 2_f64.powf(retries as f64) * 100.0; - if retries > 0 { - tracing::error!("connection thread failed, retrying in {}ms...", backoff); - tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await; - } - loop { - tracing::info!("Attempting to connect to server at {server_url}..."); - - let connection; - if server_url.starts_with("ws://") { - let (stream, _) = loop { - // Try to connect to a peer - let res = tokio_tungstenite::connect_async(server_url.clone()).await; - if let Err(e) = res { - tracing::error!("error connecting via WebSockets: {:?}", e); - // sleep for 1 second - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - continue; - } - break res.unwrap(); - }; - - connection = repo_handle_clone - .connect_tungstenite(stream, ConnDirection::Outgoing).unwrap(); - } - else { - let stream = loop { - // Try to connect to a peer - let res = TcpStream::connect(server_url.clone()).await; - if let Err(e) = res { - tracing::error!("error connecting via TCP: {:?}", e); - // sleep for 1 second - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - continue; - } - break res.unwrap(); - }; - - connection = repo_handle_clone - .connect_tokio_io(stream, ConnDirection::Outgoing).unwrap(); - } - - tracing::info!("Connected successfully!"); - - if let Err(e) = connection.handshake_complete().await { - tracing::error!("Failed to connect: {}", e); - connection_thread_tx.unbounded_send(format!("{}", e)).unwrap(); - // sleep for 5 seconds - tokio::time::sleep(std::time::Duration::from_secs(5)).await; - continue; - } - server_connected.swap(true, Ordering::Relaxed); - let completed = connection.finished().await; - server_connected.swap(false, Ordering::Relaxed); - tracing::error!("connection terminated because of: {}", completed); - connection_thread_tx.unbounded_send(format!("{}", completed)).unwrap(); - } - }); - } - fn spawn_driver_task( &self, mut rx: UnboundedReceiver, @@ -362,7 +231,6 @@ impl ProjectDriver { ) -> JoinHandle<()> { let repo_handle = self.repo_handle.clone(); let user_name = user_name.clone(); - let server_connected = self.server_connected.clone(); // let filter = EnvFilter::new("info").add_directive("automerge_repo=info".parse().unwrap()); // if let Err(e) = tracing_subscriber::registry() diff --git a/rust/src/project/sync_automerge_to_fs.rs b/rust/src/project/sync_automerge_to_fs.rs new file mode 100644 index 00000000..7ad95a7a --- /dev/null +++ b/rust/src/project/sync_automerge_to_fs.rs @@ -0,0 +1,139 @@ +use std::path::PathBuf; + +use futures::future::join_all; + +use crate::{ + fs::{file_utils::FileContent, file_utils::FileSystemEvent}, + project::branch_db::{BranchDb, HistoryRef}, +}; + +#[derive(Debug)] +pub struct SyncAutomergeToFileSystem { + branch_db: BranchDb, +} + +impl SyncAutomergeToFileSystem { + /// Create a new instance of [SyncAutomergeToFileSystem]. Does not start any process. + /// Call checkout_ref to do something. + pub fn new(branch_db: BranchDb) -> Self { + Self { branch_db } + } + + // TODO: We should consider running partial checkouts to the FS. + // Currently, if we get a remote change, and a single file is unsaved in Godot, we can't call this method at all. + // Ideally, we'd check out the synced ref, and just exclude the edited files. + + /// Check out a [HistoryRef] from the Patchwork history, changing the filesystem as necessary. + /// Returns a vector of file changes. + pub async fn checkout_ref(&self, goal_ref: HistoryRef) -> Vec { + // Ensure that there's no way anything can grab the ref while we're trying to write it + let r = self.branch_db.get_checked_out_ref_mut().await; + let mut checked_out_ref = r.write().await; + + if checked_out_ref.as_ref().is_some_and(|r| r == &goal_ref) { + return Vec::new(); + } + + let Some(changes) = self + .branch_db + .get_changed_file_content_between_refs(checked_out_ref.as_ref(), &goal_ref, false) + .await + else { + tracing::error!( + "Couldn't get changed file content between refs; canceling ref checkout of {:?}", + goal_ref + ); + return Vec::new(); + }; + + // TODO (Lilith): IMPORTANT: We need to test against known hashes of the files in the fs, to avoid writing things when they haven't actually changed. + // The old code does this by maintaining a list of file hashes that are kept up to date, but to be honest, I DON'T believe that it's that reliable. + // I think that would go out of sync constantly. + // Instead, I think we should read and hash the files before we write them. We can see how slow that is. So I must profile this. + // Consider instead using a Tokio join set here... + let futures = changes.into_iter().map(async |change| { + let written = match &change { + FileSystemEvent::FileCreated(path, content) => { + self.handle_file_update(path, content).await + } + FileSystemEvent::FileModified(path, content) => { + self.handle_file_update(path, content).await + } + FileSystemEvent::FileDeleted(path) => self.handle_file_delete(path).await, + }; + (change, written) + }); + + let results = join_all(futures) + .await + .into_iter() + .filter_map(|(event, written)| written.then_some(event)) + .collect(); + + *checked_out_ref = Some(goal_ref); + + results + } + + /// Update a file on disk if it exists and hasn't been ignored, and if the hash has changed. + /// Returns true if we successfully wrote the file. + async fn handle_file_update(&self, path: &PathBuf, content: &FileContent) -> bool { + // Skip if path matches any ignore pattern + if self.branch_db.should_ignore(&path) { + return false; + } + + let hash = content.to_hash(); + let existing_hash = match tokio::fs::read(path.clone()).await { + Ok(existing_hash) => existing_hash, + Err(e) => { + tracing::error!( + "Couldn't get existing hash for file {:?} during checkout: {}", + path, + e + ); + return false; + } + }; + + // This is a little weird because in the old system, we'd check our stored file hash DB. + // Right now, we just check to see if the files are identical before merging. + // We could consider moving to that system again. It would involve creating a separate + // file_db module that fs syncing tasks can access and lock on. + // The disadvantage to that (as well as the old system): Maintaining a separate virtual + // representation of a file system is horrifying, because if the watcher ever fucks up, + // things go out of sync and there's no way to tell without re-reading the entire directory! + if md5::compute(existing_hash) == hash { + tracing::info!( + "Skipping writing file {:?} because the hash is the same.", + path + ); + return false; + } + + // Write the file content to disk + if let Err(e) = content.write(&path).await { + tracing::error!("Failed to write file {:?} during checkout: {}", path, e); + return false; + }; + true + } + + /// Delete a file on disk, if it exists and isn't ignored. Returns true if we successfully deleted the file. + async fn handle_file_delete(&self, path: &PathBuf) -> bool { + // Skip if path matches any ignore pattern + if self.branch_db.should_ignore(&path) { + return false; + } + + // Delete the file from disk + match tokio::fs::remove_file(&path.canonicalize().unwrap()).await { + Err(e) => { + tracing::error!("Failed to delete file {:?} during checkout: {}", path, e); + return false; + } + Ok(_) => (), + }; + return true; + } +} diff --git a/rust/src/project/sync_fs_to_automerge.rs b/rust/src/project/sync_fs_to_automerge.rs new file mode 100644 index 00000000..9ae36b53 --- /dev/null +++ b/rust/src/project/sync_fs_to_automerge.rs @@ -0,0 +1,195 @@ +use std::{path::PathBuf, sync::Arc}; + +use futures::StreamExt; +use tokio::{ + sync::Mutex, + task::{JoinHandle, JoinSet}, +}; + +use crate::{ + fs::{file_utils::FileContent, file_utils::FileSystemEvent}, + project::{branch_db::BranchDb, fs_watcher::FileSystemWatcher}, +}; + +/// Tracks changes using [FileSystemWatcher], handles the changes, and tracks them as pending. +/// Call `commit` to commit them. +#[derive(Debug)] +pub struct SyncFileSystemToAutomerge { + // Collects a list of pending changes from the filesystem. + // In process, we commit these. We do this to make sure we don't make a separate commit for every file change. + // Or maybe that's OK? + // TODO (Lilith) Maybe do stream instead? This works for now though + // Stream is good though because I ***think*** we can poll with now_or_never + pending_changes: Arc>>, + branch_db: BranchDb, + handle: JoinHandle<()>, +} + +impl Drop for SyncFileSystemToAutomerge { + fn drop(&mut self) { + self.handle.abort(); + } +} + +impl SyncFileSystemToAutomerge { + pub fn new(branch_db: BranchDb) -> Self { + let pending_changes = Arc::new(Mutex::new(Vec::new())); + let pending_changes_clone = pending_changes.clone(); + let branch_db_clone = branch_db.clone(); + let handle = tokio::spawn(async move { + let changes = FileSystemWatcher::start_watching( + branch_db_clone.get_project_dir().clone(), + branch_db_clone.clone(), + ) + .await; + tokio::pin!(changes); + while let Some(event) = changes.next().await { + let (path, content) = match event { + FileSystemEvent::FileCreated(path, content) => (path, content), + FileSystemEvent::FileModified(path, content) => (path, content), + FileSystemEvent::FileDeleted(path) => (path, FileContent::Deleted), + }; + pending_changes_clone + .lock() + .await + .push((branch_db_clone.localize_path(&path), content)); + } + }); + + Self { + pending_changes, + handle, + branch_db, + } + } + + /// Make a commit of all watched, pending changes from the filesystem to automerge. + pub async fn commit(&self) { + // Because we always change the checked out ref after committing, we need to lock this in write mode. + let r = self.branch_db.get_checked_out_ref_mut().await; + let mut checked_out_ref = r.write().await; + + let mut pending_changes = self.pending_changes.lock().await; + + if pending_changes.is_empty() { + return; + } + + // If the checked-out ref is invalid, we can't commit to the current branch. + if checked_out_ref.as_ref().is_none_or(|r| !r.is_valid()) { + return; + } + + let new_ref = self + .branch_db + .commit_fs_changes( + pending_changes.clone(), + &checked_out_ref.as_ref().unwrap(), + None, + false, + ) + .await; + if let Some(new_ref) = new_ref { + pending_changes.clear(); + *checked_out_ref = Some(new_ref); + } else { + tracing::error!("Could not commit pending files! Making no changes."); + } + } + + /// Make an initial commit of ALL files from the filesystem to automerge. + pub async fn checkin(&self) { + // Because we always change the checked out ref after committing, we need to lock this in write mode. + let r = self.branch_db.get_checked_out_ref_mut().await; + let mut checked_out_ref = r.write().await; + + let files = self.get_all_files().await; + + let new_ref = self + .branch_db + .commit_fs_changes( + files.clone(), + &checked_out_ref.as_ref().unwrap(), + None, + false, + ) + .await; + + if let Some(new_ref) = new_ref { + *checked_out_ref = Some(new_ref); + } else { + tracing::error!("Could not check in files! Making no changes."); + } + // The original code runs a sync here... I'm not sure why. The equivalent in the new system would be SyncAutomergeToFileSystem. + // I think because we're updating the heads, we're OK... + // self.sync_files_at( + // branch_state.doc_handle.clone(), + // files.into_iter().map(|(path, content)| (PathBuf::from(path), content)).collect::>(), + // Some(branch_state.synced_heads.clone()), + } + + fn get_all_files_recur( + branch_db: BranchDb, + path: PathBuf, + content: Arc>>>, + ) -> impl Future> + Send { + async move { + let mut dir = tokio::fs::read_dir(path).await.ok()?; + let mut set = JoinSet::new(); + while let Some(entry) = dir.next_entry().await.ok()? { + let path = entry.path(); + + // Skip if path matches any ignore pattern + if branch_db.should_ignore(&path) { + continue; + } + + if path.is_file() { + let path_clone = path.clone(); + let content = content.clone(); + set.spawn(async move { + let data = tokio::fs::read(path).await; + match data { + Ok(data) => content + .lock() + .await + .as_mut() + .unwrap() + .push((path_clone, FileContent::from_buf(data))), + Err(e) => tracing::error!("Error while trying to read file: {}", e), + } + None + }); + } else if path.is_dir() { + let path = path.clone(); + let branch_db = branch_db.clone(); + let content = content.clone(); + set.spawn( + async move { Self::get_all_files_recur(branch_db, path, content).await }, + ); + } + } + while let Some(_) = set.join_next().await {} + None + } + } + + async fn get_all_files(&self) -> Vec<(String, FileContent)> { + let content = Arc::new(Mutex::new(Some(Vec::new()))); + Self::get_all_files_recur( + self.branch_db.clone(), + self.branch_db.get_project_dir(), + content.clone(), + ) + .await; + // steal the content from the mutex + content + .lock() + .await + .take() + .unwrap() + .into_iter() + .map(|(path, content)| (self.branch_db.localize_path(&path), content)) + .collect() + } +} From 836bdd92d7493097c7a4f3124ba49defa0d755fd Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Sat, 24 Jan 2026 04:13:10 -0800 Subject: [PATCH 02/49] more refactor --- Cargo.lock | 5 +- Cargo.toml | 1 + public/gdscript/sidebar.gd | 2 +- rust/Cargo.toml | 1 + rust/src/interop/godot_project.rs | 2 +- rust/src/project/branch_db.rs | 17 +- rust/src/project/connection.rs | 79 ++++++--- rust/src/project/document_watcher.rs | 94 +++++----- rust/src/project/driver.rs | 50 +++--- rust/src/project/fs_watcher.rs | 16 +- rust/src/project/new_project.rs | 214 ++++++++++++----------- rust/src/project/peer_watcher.rs | 43 ++--- rust/src/project/project_api_impl.rs | 16 +- rust/src/project/sync_automerge_to_fs.rs | 14 +- rust/src/project/sync_fs_to_automerge.rs | 49 ++++-- 15 files changed, 351 insertions(+), 252 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a13de22..500db02c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1277,6 +1277,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-tungstenite", + "tokio-util", "tracing", "tracing-appender", "tracing-subscriber", @@ -2049,9 +2050,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index ac60eef6..c8ab6f97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ [profile.release_debug] inherits = "release" opt-level = 0 +strip = "none" debug = true lto = false panic = "abort" diff --git a/public/gdscript/sidebar.gd b/public/gdscript/sidebar.gd index 6a954421..5e9923e5 100644 --- a/public/gdscript/sidebar.gd +++ b/public/gdscript/sidebar.gd @@ -75,7 +75,7 @@ signal reload_ui(); signal user_name_dialog_closed(); func _update_ui_on_state_change(): - print("Patchwork: Updating UI due to state change...") + # print("Patchwork: Updating UI due to state change...") update_ui() func _update_ui_on_branch_checked_out(): diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 7776b62b..e650f458 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -59,6 +59,7 @@ regex = "1.12.2" tokio-tungstenite = "0.27.0" async-stream = "0.3.6" tokio-stream = { version = "0.1.18", features = ["sync"] } +tokio-util = "0.7.18" [build-dependencies] cbindgen = "^0.27" diff --git a/rust/src/interop/godot_project.rs b/rust/src/interop/godot_project.rs index e00b9a6d..ce6b6994 100644 --- a/rust/src/interop/godot_project.rs +++ b/rust/src/interop/godot_project.rs @@ -414,7 +414,7 @@ impl GodotProject { if !self.pending_editor_update.any_changes() { return false; } - if !Driver::safe_to_update_godot() { + if !Project::safe_to_update_godot() { return false; } self.base_mut().set_process(false); diff --git a/rust/src/project/branch_db.rs b/rust/src/project/branch_db.rs index e9f83f01..fc01dc18 100644 --- a/rust/src/project/branch_db.rs +++ b/rust/src/project/branch_db.rs @@ -1,22 +1,19 @@ use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, path::PathBuf, sync::Arc, }; -use automerge::{Automerge, ChangeHash, ObjId, ObjType, ROOT, ReadDoc}; -use autosurgeon::Doc; +use automerge::ChangeHash; use samod::{DocHandle, DocumentId, Repo}; use serde::{Deserialize, Serialize}; use tokio::sync::{Mutex, RwLock}; -use crate::{ - fs::file_utils::FileContent, - helpers::{ - branch::{BinaryDocState, BranchState, BranchesMetadataDoc}, - utils::{CommitMetadata, commit_with_attribution_and_timestamp}, - }, -}; +use crate:: + helpers:: + branch::{BinaryDocState, BranchState, BranchesMetadataDoc} + +; mod branch; mod commit; diff --git a/rust/src/project/connection.rs b/rust/src/project/connection.rs index e1191132..73c7a97a 100644 --- a/rust/src/project/connection.rs +++ b/rust/src/project/connection.rs @@ -1,17 +1,30 @@ +use std::{fmt::Display, sync::Arc}; + use futures::{Stream, StreamExt as _}; use samod::{ConnDirection, ConnFinishedReason, Repo}; use tokio::{ - net::TcpStream, - sync::{broadcast, watch}, - task::JoinHandle, + net::TcpStream, select, sync::{broadcast, watch} }; use tokio_stream::wrappers::BroadcastStream; +use tokio_util::sync::CancellationToken; #[derive(Debug, Clone)] enum ConnectionStoppedReason { TcpConnectionError(String), WebSocketsConnectionError(String), ConnectionCompleted(ConnFinishedReason), + ConnectionCancelled() +} + +impl Display for ConnectionStoppedReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TcpConnectionError(e) => write!(f, "TCP connection error: {}", e), + Self::WebSocketsConnectionError(e) => write!(f, "WebSocket connection error: {}", e), + Self::ConnectionCompleted(reason) => write!(f, "Connection completed: {:?}", reason), + Self::ConnectionCancelled() => write!(f, "Connection cancelled"), + } + } } #[derive(Debug, Clone)] @@ -27,12 +40,12 @@ pub enum RemoteConnectionStatus { Disconnected, } -/// Connects a repo to the remote server. +/// Connects a repo to the remote server. Shuts down when dropped. #[derive(Debug)] pub struct RemoteConnection { - connection_handle: JoinHandle<()>, status_rx: watch::Receiver, events_tx: broadcast::Sender, + inner: Arc } #[derive(Clone, Debug)] @@ -41,12 +54,13 @@ struct RemoteConnectionInner { server_url: String, events_tx: broadcast::Sender, status_tx: watch::Sender, + token: CancellationToken } impl Drop for RemoteConnection { // Stop the connection on drop fn drop(&mut self) { - &self.connection_handle.abort(); + self.inner.token.cancel() } } @@ -56,21 +70,23 @@ impl RemoteConnection { pub fn new(repo: Repo, server_url: String) -> Self { let (events_tx, _) = broadcast::channel(32); let (status_tx, status_rx) = watch::channel(RemoteConnectionStatus::Disconnected); - let inner = RemoteConnectionInner { + let inner = Arc::new(RemoteConnectionInner { repo, server_url, events_tx: events_tx.clone(), status_tx, - }; + token: CancellationToken::new() + }); - let connection_handle: JoinHandle<()> = tokio::spawn(async move { - inner.retry_connection().await; + let inner_clone = inner.clone(); + tokio::spawn(async move { + inner_clone.retry_connection().await; }); Self { - connection_handle, + inner, status_rx, - events_tx, + events_tx } } @@ -103,9 +119,20 @@ impl RemoteConnectionInner { let backoff = 1000; loop { let termination_reason = self.try_connection().await; + match termination_reason { + ConnectionStoppedReason::ConnectionCancelled() => break, + _ => (), + } tracing::error!("Connection failure: {:?}", termination_reason); tracing::error!("Retrying in {}ms...", backoff); - tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await; + + // Look for the cancelation token here as well, so we cancel from the backoff + select! { + _ = tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)) => {} + _ = self.token.cancelled() => { + break; + } + } } } @@ -120,7 +147,7 @@ impl RemoteConnectionInner { let res = tokio_tungstenite::connect_async(server_url.clone()).await; match res { Err(e) => { - self.events_tx.send(RemoteConnectionEvent::ConnectionFailed); + _ = self.events_tx.send(RemoteConnectionEvent::ConnectionFailed); return ConnectionStoppedReason::WebSocketsConnectionError(e.to_string()); } Ok((res, _)) => repo_handle @@ -133,7 +160,7 @@ impl RemoteConnectionInner { let res = TcpStream::connect(server_url.clone()).await; match res { Err(e) => { - self.events_tx.send(RemoteConnectionEvent::ConnectionFailed); + _ = self.events_tx.send(RemoteConnectionEvent::ConnectionFailed); return ConnectionStoppedReason::TcpConnectionError(e.to_string()); } Ok(res) => repo_handle @@ -145,18 +172,24 @@ impl RemoteConnectionInner { tracing::info!("Connected successfully!"); if let Err(e) = connection.handshake_complete().await { - self.status_tx.send(RemoteConnectionStatus::Disconnected); - self.events_tx.send(RemoteConnectionEvent::ConnectionFailed); + _ = self.status_tx.send(RemoteConnectionStatus::Disconnected); + _ = self.events_tx.send(RemoteConnectionEvent::ConnectionFailed); return ConnectionStoppedReason::ConnectionCompleted(e); } tracing::info!("Handshake completed!"); - self.status_tx.send(RemoteConnectionStatus::Connected); - self.events_tx.send(RemoteConnectionEvent::Connected); + _ = self.status_tx.send(RemoteConnectionStatus::Connected); + _ = self.events_tx.send(RemoteConnectionEvent::Connected); - let completed = connection.finished().await; - self.status_tx.send(RemoteConnectionStatus::Disconnected); - self.events_tx.send(RemoteConnectionEvent::ConnectionCompleted); - return ConnectionStoppedReason::ConnectionCompleted(completed); + select! { + completed = connection.finished() => { + _ = self.status_tx.send(RemoteConnectionStatus::Disconnected); + _ = self.events_tx.send(RemoteConnectionEvent::ConnectionCompleted); + return ConnectionStoppedReason::ConnectionCompleted(completed); + }, + _ = self.token.cancelled() => { + return ConnectionStoppedReason::ConnectionCancelled(); + } + } } } diff --git a/rust/src/project/document_watcher.rs b/rust/src/project/document_watcher.rs index 8c9cd0ac..baa63a17 100644 --- a/rust/src/project/document_watcher.rs +++ b/rust/src/project/document_watcher.rs @@ -19,58 +19,67 @@ use automerge::{ChangeHash, ROOT, ReadDoc}; use autosurgeon::hydrate; use futures::{FutureExt, StreamExt}; use samod::{DocHandle, DocumentId, Repo}; -use tokio::{sync::Mutex, task::JoinHandle}; +use tokio::select; +use tokio_util::sync::CancellationToken; /// Tracks branch and metadata documents from an Automerge repo, updating BranchDB when the state changes. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct DocumentWatcher { + inner: Arc, +} + +#[derive(Debug, Clone)] +struct DocumentWatcherInner { repo: Repo, branch_db: BranchDb, - tracking_handles: Arc>>>, + token: CancellationToken, } impl Drop for DocumentWatcher { fn drop(&mut self) { - for handle in self.tracking_handles.blocking_lock().drain(..) { - // NOTE: spawn_blocking calls are not immediately aborted. They will run to completion in the background, then finish. - handle.abort(); - } + self.inner.token.cancel(); } } impl DocumentWatcher { /// Spawns the [DocumentWatcher], creating parallel tasks for the metadata document tracking and subsequent tasks for any child documents. - pub fn new(repo: Repo, branch_db: BranchDb, metadata_handle: DocHandle) -> Self { - let this = Self { - repo, + pub async fn new(repo: Repo, branch_db: BranchDb, metadata_handle: DocHandle) -> Self { + let inner = Arc::new(DocumentWatcherInner { branch_db, - tracking_handles: Arc::new(Mutex::new(Vec::new())), - }; - - let this_clone = this.clone(); - - this.tracking_handles - .blocking_lock() - .push(tokio::spawn(async move { - // do the initial ingest - this_clone - .ingest_metadata_document(metadata_handle.clone()) - .await; - // track changes for future ingests - this_clone.track_metadata_document(metadata_handle).await; - })); - - return this; + repo, + token: CancellationToken::new(), + }); + + let inner_clone = inner.clone(); + + tokio::spawn(async move { + // do the initial ingest + inner_clone + .ingest_metadata_document(metadata_handle.clone()) + .await; + // track changes for future ingests + inner_clone.track_metadata_document(metadata_handle).await; + }); + + return Self { inner }; } +} +impl DocumentWatcherInner { // The branch documents are a document for each branch, containing all the serialized data for all scenes and text files. async fn track_branch_document(&self, handle: DocHandle) { let mut stream = handle.changes(); loop { - let _ = stream.next().await; - // collapse the rest of the stream, in case multiple futures are ready - while let Some(_) = stream.next().now_or_never().flatten() {} - self.ingest_branch_document(handle.clone()).await; + select! { + _ = stream.next() => { + // collapse the rest of the stream, in case multiple futures are ready + while let Some(_) = stream.next().now_or_never().flatten() {} + self.ingest_branch_document(handle.clone()).await; + }, + _ = self.token.cancelled() => { + break; + } + } } } @@ -78,10 +87,16 @@ impl DocumentWatcher { async fn track_metadata_document(&self, handle: DocHandle) { let mut stream = handle.changes(); loop { - let _ = stream.next().await; - // collapse the rest of the stream, in case multiple futures are ready - while let Some(_) = stream.next().now_or_never().flatten() {} - self.ingest_metadata_document(handle.clone()).await; + select! { + _ = stream.next() => { + // collapse the rest of the stream, in case multiple futures are ready + while let Some(_) = stream.next().now_or_never().flatten() {} + self.ingest_metadata_document(handle.clone()).await; + }, + _ = self.token.cancelled() => { + break; + } + } } } @@ -204,7 +219,8 @@ impl DocumentWatcher { .await .unwrap(); self.branch_db - .set_metadata_state(handle.document_id().clone(), meta.clone()); + .set_metadata_state(handle.document_id().clone(), meta.clone()) + .await; // check if there are new branches that haven't loaded yet for (branch_id_str, _) in meta.branches.iter() { let branch_id = DocumentId::from_str(branch_id_str).unwrap(); @@ -220,11 +236,7 @@ impl DocumentWatcher { self.ingest_branch_document(handle.clone()).await; // Track the document let this = self.clone(); - self.tracking_handles - .blocking_lock() - .push(tokio::spawn(async move { - this.track_branch_document(handle).await - })); + tokio::spawn(async move { this.track_branch_document(handle).await }); } } } diff --git a/rust/src/project/driver.rs b/rust/src/project/driver.rs index 5867efa9..e0157f31 100644 --- a/rust/src/project/driver.rs +++ b/rust/src/project/driver.rs @@ -19,16 +19,19 @@ use std::sync::Arc; /// The main driver for the project. /// Hooks together all the various controllers. /// When this object is constructed, it is started. When the handle is dropped, it shuts down. -#[derive(Clone, Debug)] +#[derive(Debug, Clone)] pub struct Driver { inner: Arc, } #[derive(Debug)] pub struct DriverInner { + repo: Repo, + #[allow(unused)] connection: RemoteConnection, branch_db: BranchDb, peer_watcher: PeerWatcher, + #[allow(unused)] document_watcher: DocumentWatcher, sync_automerge_to_fs: SyncAutomergeToFileSystem, sync_fs_to_automerge: SyncFileSystemToAutomerge, @@ -37,6 +40,17 @@ pub struct DriverInner { // differ: Differ, } +impl Drop for DriverInner { + fn drop(&mut self) { + // don't use tokio for this, I think that's correct? + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.block_on(self.repo.stop()); + } else { + futures::executor::block_on(self.repo.stop()); + } + } +} + impl Driver { async fn parse_gitignore( project_dir: &PathBuf, @@ -47,7 +61,7 @@ impl Driver { let content = match tokio::fs::read_to_string(gitignore_path).await { Ok(content) => content, Err(_) => { - tracing::error!("Couldn't read gitignore file at {:?}", gitignore_path); + tracing::info!("Couldn't read gitignore file at {:?}", gitignore_path); return ignore_globs; } }; @@ -91,15 +105,6 @@ impl Driver { storage_directory: PathBuf, metadata_id: Option, ) -> Option { - // TODO (Lilith): ensure we make this work across the ENTIRE program. Initialize it only once, etc. - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(1) - .enable_all() - .thread_name("GodotProjectDriver: worker thread") - .build() - .unwrap(); - - let _ = runtime.enter(); let storage = samod::storage::TokioFilesystemStorage::new(storage_directory); let repo = Repo::build_tokio() .with_concurrency(ConcurrencyConfig::Threadpool( @@ -158,7 +163,7 @@ impl Driver { // The document watcher will auto-ingest the provided metadata handle. let document_watcher = - DocumentWatcher::new(repo.clone(), branch_db.clone(), metadata_handle); + DocumentWatcher::new(repo.clone(), branch_db.clone(), metadata_handle).await; let peer_watcher = PeerWatcher::new(repo.clone(), branch_db.clone()); let sync_automerge_to_fs = SyncAutomergeToFileSystem::new(branch_db.clone()); let sync_fs_to_automerge = SyncFileSystemToAutomerge::new(branch_db.clone()); @@ -169,6 +174,7 @@ impl Driver { Some(Driver { inner: DriverInner { + repo, connection, branch_db, peer_watcher, @@ -320,13 +326,6 @@ impl Driver { .map(|(id, _)| id) } - pub fn safe_to_update_godot() -> bool { - return !(EditorFilesystemAccessor::is_scanning() - || PatchworkEditorAccessor::is_editor_importing() - || PatchworkEditorAccessor::is_changing_scene() - || PatchworkEditorAccessor::unsaved_files_open()); - } - pub async fn get_main_branch(&self) -> Option { self.inner .branch_db @@ -413,6 +412,13 @@ impl Driver { changes .into_iter() + // only changes on the current branch with valid metadata + .filter(|change| { + change + .metadata + .as_ref() + .is_some_and(|m| m.branch_id.as_ref().is_some_and(|id| id == doc_id)) + }) .enumerate() .map(|(i, change)| CommitInfo { synced: (i as i32) <= synced_until_index, @@ -430,7 +436,7 @@ impl Driver { /// Make sure not to run two of these at once, for safety. /// (It would likely be OK state-wise but weird results might happen on the UI side.) /// Returns a vector of filesystem changes we performed. - pub async fn sync(&self) -> Vec { + pub async fn sync(&self, safe_to_update_godot: bool) -> Vec { // TODO (Lilith): There are inefficiencies with this strategy. // Basically, every time we save a file, it'll do a bunch of extra work. // It will first commit the changes, then it will check out the changes we just committed. @@ -438,7 +444,7 @@ impl Driver { // The same happens in reverse: when we check out a ref, it will attempt to commit and not // find any actual changes. // Maybe that's OK, we need to profile to see if it's a problem. - let changes = if Self::safe_to_update_godot() { + let changes = if safe_to_update_godot { let goal_ref = { let current_ref_lock = self.inner.branch_db.get_checked_out_ref_mut().await; let current_ref_guard = current_ref_lock.read().await; @@ -466,7 +472,7 @@ impl Driver { // guard dropped here }; // TODO (Lilith): minor problem, the checked out ref could change between these lines when the guard is dropped. - // That said, uhhhh I think that's fine? We're already syncing, so other sync methods shouldn't effect it. + // That said, uhhhh I think that's fine? We're already syncing, so other sync methods shouldn't affect it. // If we're doing some branch edits, like a merge or revert preview or something, that could mean we // never checkout our desired ref... but we need to rethink this for branch swapping anyways. // For branch swapping, we either need to checkout a ref elsewhere during a merge or something and prevent sync. diff --git a/rust/src/project/fs_watcher.rs b/rust/src/project/fs_watcher.rs index 433921f5..163dfaa7 100644 --- a/rust/src/project/fs_watcher.rs +++ b/rust/src/project/fs_watcher.rs @@ -13,7 +13,7 @@ use notify_debouncer_mini::{DebouncedEvent, new_debouncer_opt}; use tokio::{ sync::{ Mutex, - mpsc::{self}, + mpsc::{self, UnboundedSender}, }, task::JoinSet, time::sleep, @@ -40,7 +40,7 @@ pub struct FileSystemWatcher { watch_path: PathBuf, file_hashes: Arc>>, branch_db: BranchDb, - found_ignored_paths: Arc>>, + found_ignored_paths: Arc>> } impl FileSystemWatcher { @@ -88,7 +88,7 @@ impl FileSystemWatcher { // Initialize the hash map with existing files async fn initialize_file_hashes(&self) { self.initialize_file_hashes_recur(self.watch_path.clone()) - .await; + .await.unwrap(); } // Handle file creation and modification events @@ -185,10 +185,11 @@ impl FileSystemWatcher { .with_batch_mode(true) .with_notify_config(notify_config); + let notify_tx_clone = notify_tx.clone(); let mut debouncer = new_debouncer_opt::<_, RecommendedWatcher>( debouncer_config, move |event: Result, notify::Error>| { - notify_tx.send(event).unwrap(); + notify_tx_clone.send(event).unwrap(); }, ) .unwrap(); @@ -212,19 +213,24 @@ impl FileSystemWatcher { let _ret = debouncer.watcher().unwatch(path); } let stream = UnboundedReceiverStream::new(notify_rx); - // Process both file system events and update events + // Process both file system events and update eventss stream! { + // move the debouncer into the returned stream + let _keep_alive = debouncer; // Handle file system events for await notify_events in stream { let Ok(notify_events) = notify_events else { continue; }; + tracing::debug!("Heard filesystem event list of {:?} items", notify_events.len()); for notify_event in notify_events { + tracing::debug!("Heard filesystem event {:?}", notify_event); if let Some(evt) = this.process_notify_event(notify_event).await { yield evt; } } } + tracing::debug!("fs_watcher shutting down!"); } } } diff --git a/rust/src/project/new_project.rs b/rust/src/project/new_project.rs index 173c8031..24d21538 100644 --- a/rust/src/project/new_project.rs +++ b/rust/src/project/new_project.rs @@ -1,17 +1,16 @@ use crate::diff::differ::ProjectDiff; use crate::fs::file_utils::FileSystemEvent; use crate::helpers::utils::{CommitInfo, summarize_changes}; -use crate::interop::godot_accessors:: - PatchworkConfigAccessor -; +use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkConfigAccessor, PatchworkEditorAccessor}; use crate::project::branch_db::HistoryRef; use crate::project::driver::Driver; use crate::project::project_api::ChangeViewModel; use automerge::ChangeHash; +use futures::future::join_all; use samod::DocumentId; +use std::cell::RefCell; use std::path::PathBuf; use std::time::SystemTime; -use std::cell::RefCell; use std::{collections::HashMap, str::FromStr}; use tokio::runtime::Runtime; @@ -29,7 +28,7 @@ pub struct Project { pub(super) changes: HashMap, last_ingest: (SystemTime, i32), ingest_requested: bool, - last_known_branch: Option, + last_known_branch: Option, // Cached diffs between refs pub(super) diff_cache: RefCell>, @@ -63,69 +62,80 @@ impl Project { changes: HashMap::new(), last_ingest: (SystemTime::UNIX_EPOCH, 0), ingest_requested: true, - last_known_branch: None, + last_known_branch: None, diff_cache: RefCell::new(HashMap::new()), } } /// Expensive operation to ingest all branch changes from automerge into the project data. /// Should be called when we think there are new changes to process. - async fn ingest_changes(&mut self) { - let Some(driver) = self.driver.clone() else { - tracing::error!("Driver not started, can't ingest changes!"); - return; - }; + fn ingest_changes(&mut self) { + let Some(driver) = self.driver.clone() else { + tracing::error!("Driver not started, can't ingest changes!"); + return; + }; - tracing::info!("Ingesting changes..."); + // tracing::info!("Ingesting changes..."); + + let changes = self + .runtime + .block_on(self.runtime.spawn(async move { + let changes = + driver + .get_changes() + .await + .into_iter() + .map(async |change| CommitInfo { + summary: Self::get_change_summary(&change, driver.clone()) + .await + .unwrap_or("Invalid data".to_string()), + ..change + }); + // a little awk. Parallelization doesn't really matter here at all and is maybe bad (bc locks). + // but otherwise we're iteratively constructing a vec, probably doesn't matter + join_all(changes).await + })) + .unwrap(); - let changes = self.runtime.block_on(tokio::spawn(async move { - driver.get_changes().await - })).unwrap(); + self.history.clear(); + self.changes.clear(); // Consume changes into self.changes for change in changes { self.history.push(change.hash); - self.changes.insert(change.hash, CommitInfo { - // the only actual things we're awaiting here are data locks - // so dw about parallelization - summary: self.get_change_summary(&change).await.unwrap_or("Invalid data".to_string()), - ..change - }); + self.changes.insert(change.hash, change); } } - async fn get_change_summary(&self, change: &CommitInfo) -> Option { - let meta = change.metadata.as_ref(); - let author = meta?.username.as_ref()?; - - // merge commit - if let Some(merge_info) = &meta?.merge_metadata { - let merged_branch = &self - .driver - .as_ref() - .unwrap() // the driver had better be valid - .get_branch_name(&merge_info.merged_branch_id.clone()) - .await - .unwrap_or(merge_info.merged_branch_id.to_string()); - return Some(format!("↪ {author} merged {merged_branch}")); - } - - // revert commit - if let Some(revert_info) = &meta?.reverted_to { - let heads = revert_info - .iter() - .map(|s| &s[..7]) - .collect::>() - .join(", "); - return Some(format!("↩ {author} reverted to {heads}")); - } - - // initial commit - if change.is_setup() { - return Some(format!("Initialized repository")); - } - - return Some(summarize_changes(&author, meta?.changed_files.as_ref()?)); + async fn get_change_summary(change: &CommitInfo, driver: Driver) -> Option { + let meta = change.metadata.as_ref(); + let author = meta?.username.clone().unwrap_or("Anonymous".to_string()); + + // merge commit + if let Some(merge_info) = &meta?.merge_metadata { + let merged_branch = driver + .get_branch_name(&merge_info.merged_branch_id.clone()) + .await + .unwrap_or(merge_info.merged_branch_id.to_string()); + return Some(format!("↪ {author} merged {merged_branch}")); + } + + // revert commit + if let Some(revert_info) = &meta?.reverted_to { + let heads = revert_info + .iter() + .map(|s| &s[..7]) + .collect::>() + .join(", "); + return Some(format!("↩ {author} reverted to {heads}")); + } + + // initial commit + if change.is_setup() { + return Some(format!("Initialized repository")); + } + + return Some(summarize_changes(&author, meta?.changed_files.as_ref()?)); } /// Request for a change ingestion to be dispatched. @@ -162,11 +172,7 @@ impl Project { true } - pub fn get_cached_diff( - &self, - before: HistoryRef, - after: HistoryRef, - ) -> ProjectDiff { + pub fn get_cached_diff(&self, before: HistoryRef, after: HistoryRef) -> ProjectDiff { self.diff_cache .borrow_mut() .entry((before.clone(), after.clone())) @@ -178,23 +184,27 @@ impl Project { self.diff_cache.borrow_mut().clear(); } - pub fn get_diff( - &self, - before: HistoryRef, - after: HistoryRef, - ) -> ProjectDiff { + // Do not run this on anything except the main thread! + pub fn safe_to_update_godot() -> bool { + return !(EditorFilesystemAccessor::is_scanning() + || PatchworkEditorAccessor::is_editor_importing() + || PatchworkEditorAccessor::is_changing_scene() + || PatchworkEditorAccessor::unsaved_files_open()); + } + + pub fn get_diff(&self, before: HistoryRef, after: HistoryRef) -> ProjectDiff { let Some(driver) = &self.driver else { return ProjectDiff::default(); }; - ProjectDiff::default() - - // TODO (Lilith): make this code work; ProjectDiff includes variants which can't be pushed across threads - // TODO: these can get expensive; propagate a spinner to the UI - // self.runtime.block_on(async move { - // tokio::spawn(async move { - // driver.get_diff(&before, &after).await - // }).await.unwrap() - // }) + ProjectDiff::default() + + // TODO (Lilith): make this code work; ProjectDiff includes variants which can't be pushed across threads + // TODO: these can get expensive; propagate a spinner to the UI + // self.runtime.block_on( + // self.runtime.spawn(async move { + // driver.get_diff(&before, &after).await + // }).await.unwrap() + // ) } pub fn start(&mut self) { @@ -220,8 +230,8 @@ impl Project { } }; - // If the metadata ID is not a valid document ID, give up. - // If it's an empty string, returns None so we can make a new doc. + // If the metadata ID is not a valid document ID, give up. + // If it's an empty string, returns None so we can make a new doc. let metadata_id = match Some(PatchworkConfigAccessor::get_project_value( "project_doc_id", "", @@ -245,13 +255,16 @@ impl Project { metadata_id ); - let project_dir = self.project_dir.clone(); - self.driver = self.runtime.block_on(async move { - // I think it's correct to spawn this on a different task explicitly, because block_on runs the future on the current thread, not a worker thread. - tokio::spawn(async move { - Driver::new(server_url, project_dir, storage_dir, metadata_id).await - }).await.unwrap() - }); + let project_dir = self.project_dir.clone(); + self.driver = self + .runtime + .block_on( + // I think it's correct to spawn this on a different task explicitly, because block_on runs the future on the current thread, not a worker thread. + self.runtime.spawn(async move { + Driver::new(server_url, project_dir, storage_dir, metadata_id).await + }), + ) + .unwrap(); if self.driver.is_none() { tracing::error!("Could not start the driver!"); @@ -264,30 +277,35 @@ impl Project { } pub fn process(&mut self, _delta: f64) -> (Vec, Vec) { - let Some(driver) = &self.driver else { - return Default::default(); - }; - let driver = driver.clone(); + let Some(driver) = &self.driver else { + return Default::default(); + }; + let driver = driver.clone(); let mut signals: Vec = Vec::new(); if self.try_ingest_changes() { signals.push(GodotProjectSignal::ChangesIngested); } + let safe_to_update = Self::safe_to_update_godot(); + // Run the main sync + let (changed_files, checked_out_ref) = + self.runtime + .block_on(self.runtime.spawn(async move { + (driver.sync(safe_to_update).await, driver.get_checked_out_ref().await) + })) + .unwrap(); + + let current_branch = checked_out_ref.and_then(|r| Some(r.branch)); + if self.last_known_branch != current_branch { + signals.push(GodotProjectSignal::CheckedOutBranch); + self.last_known_branch = current_branch; + } - // Run the main sync - let (changed_files, checked_out_ref) = self.runtime.block_on(async move { - tokio::spawn(async move { - (driver.sync().await, driver.get_checked_out_ref().await) - }).await.unwrap() - }); - - let current_branch = checked_out_ref.and_then(|r| Some(r.branch)); - if self.last_known_branch != current_branch { - signals.push(GodotProjectSignal::CheckedOutBranch); - } + // TODO (Lilith): VERY IMPORTANT, set the patchwork config branch ID here!!! + // So that we save the branch ID for future checkouts. - // TODO (Lilith): VERY IMPORTANT, set the patchwork config branch ID here!!! - // So that we save the branch ID for future checkouts. + // TODO (Lilith): Don't request an ingestion every frame + self.request_ingestion(); (changed_files, signals) } diff --git a/rust/src/project/peer_watcher.rs b/rust/src/project/peer_watcher.rs index cd0ef32c..f69ba292 100644 --- a/rust/src/project/peer_watcher.rs +++ b/rust/src/project/peer_watcher.rs @@ -7,49 +7,44 @@ use tokio::{sync::Mutex, task::JoinHandle}; #[derive(Debug)] pub struct PeerWatcher { - repo_handle: Repo, - branch_db: BranchDb, server_info: Arc>>, - handle: Option>, + handle: JoinHandle<()>, } impl Drop for PeerWatcher { fn drop(&mut self) { // Is this safe? Alternatively we could use a cancellation token // I think it's safe though - self.handle.as_ref().map(|h| h.abort()); + self.handle.abort(); } } impl PeerWatcher { pub fn new(repo_handle: Repo, branch_db: BranchDb) -> Self { - Self { - repo_handle, - branch_db, - server_info: Arc::new(Mutex::new(None)), - handle: None, - } - } - - pub async fn get_server_info(&self) -> Option { - return self.server_info.lock().await.clone(); - } - - pub fn start(&mut self) { - let repo_handle = self.repo_handle.clone(); - let server_info = self.server_info.clone(); - self.handle = Some(tokio::spawn(async move { - let (_, stream) = repo_handle.connected_peers(); + let server_info = Arc::new(Mutex::new(None)); + let server_info_clone = server_info.clone(); + let repo_handle_clone = repo_handle.clone(); + let handle = tokio::spawn(async move { + let (_, stream) = repo_handle_clone.connected_peers(); tokio::pin!(stream); while let Some(peers) = stream.next().await { // Currently, we only ever have 1 peer: the server. // Therefore, this code expects that the server is the first and only peer, if it's connected. // When we move to more peers, we'll need to figure out a way to identify the server here. if let Some(info) = peers.first() { - Self::update_server_info(server_info.clone(), info.clone()).await; + Self::update_server_info(server_info_clone.clone(), info.clone()).await; } } - })); + }); + + Self { + server_info, + handle, + } + } + + pub async fn get_server_info(&self) -> Option { + return self.server_info.lock().await.clone(); } async fn update_server_info( @@ -58,7 +53,7 @@ impl PeerWatcher { ) { let mut server_info = old_info.lock().await; if server_info.is_none() { - server_info.insert(new_info); + *server_info = Some(new_info); return; } let mut info = server_info.clone().unwrap(); diff --git a/rust/src/project/project_api_impl.rs b/rust/src/project/project_api_impl.rs index f0b37dbc..addf5c19 100644 --- a/rust/src/project/project_api_impl.rs +++ b/rust/src/project/project_api_impl.rs @@ -31,7 +31,7 @@ impl ProjectViewModel for Project { fn get_project_id(&self) -> Option { let driver = self.driver.clone()?; self.runtime - .block_on(tokio::spawn(async move { driver.get_metadata_doc().await })) + .block_on(self.runtime.spawn(async move { driver.get_metadata_doc().await })) .unwrap() } @@ -73,9 +73,9 @@ impl ProjectViewModel for Project { let Some(driver) = self.driver.clone() else { return; }; - self.runtime.block_on(tokio::spawn(async move { + self.runtime.block_on(self.runtime.spawn(async move { driver.set_username(Some(name)).await - })); + })).unwrap(); } fn can_create_merge_preview_branch(&self) -> bool { @@ -295,7 +295,7 @@ impl ProjectViewModel for Project { let id = id.clone(); let state = self .runtime - .block_on(tokio::spawn( + .block_on(self.runtime.spawn( async move { driver.get_branch_state(&id).await }, )) .unwrap(); @@ -341,7 +341,7 @@ impl ProjectViewModel for Project { let driver = self.driver.clone()?; let id = self .runtime - .block_on(tokio::spawn(async move { driver.get_main_branch().await })) + .block_on(self.runtime.spawn(async move { driver.get_main_branch().await })) .unwrap()?; self.get_branch(&id) } @@ -350,7 +350,7 @@ impl ProjectViewModel for Project { let driver = self.driver.clone()?; let id = self .runtime - .block_on(tokio::spawn( + .block_on(self.runtime.spawn( async move { driver.get_checked_out_ref().await }, )) .unwrap()?; @@ -437,7 +437,7 @@ impl ProjectViewModel for Project { let driver = self.driver.clone()?; let branch_state = self .runtime - .block_on(tokio::spawn(async move { + .block_on(self.runtime.spawn(async move { driver .get_branch_state(&driver.get_checked_out_ref().await?.branch) .await @@ -531,7 +531,7 @@ impl ProjectViewModel for Project { let driver = self.driver.clone()?; let branch_state = self .runtime - .block_on(tokio::spawn(async move { + .block_on(self.runtime.spawn(async move { driver .get_branch_state(&driver.get_checked_out_ref().await?.branch) .await diff --git a/rust/src/project/sync_automerge_to_fs.rs b/rust/src/project/sync_automerge_to_fs.rs index 7ad95a7a..30d23cbc 100644 --- a/rust/src/project/sync_automerge_to_fs.rs +++ b/rust/src/project/sync_automerge_to_fs.rs @@ -34,6 +34,11 @@ impl SyncAutomergeToFileSystem { return Vec::new(); } + tracing::info!( + "Our current ref is different than the requested ref. Attempting to checkout {:?}", + goal_ref + ); + let Some(changes) = self .branch_db .get_changed_file_content_between_refs(checked_out_ref.as_ref(), &goal_ref, false) @@ -64,11 +69,16 @@ impl SyncAutomergeToFileSystem { (change, written) }); - let results = join_all(futures) + let results: Vec = join_all(futures) .await .into_iter() .filter_map(|(event, written)| written.then_some(event)) .collect(); + + tracing::info!( + "Wrote {:?} files!", + results.len() + ); *checked_out_ref = Some(goal_ref); @@ -116,6 +126,7 @@ impl SyncAutomergeToFileSystem { tracing::error!("Failed to write file {:?} during checkout: {}", path, e); return false; }; + tracing::info!("Successfully modified {:?}", path); true } @@ -134,6 +145,7 @@ impl SyncAutomergeToFileSystem { } Ok(_) => (), }; + tracing::info!("Successfully deleted {:?}", path); return true; } } diff --git a/rust/src/project/sync_fs_to_automerge.rs b/rust/src/project/sync_fs_to_automerge.rs index 9ae36b53..c6a7c6d7 100644 --- a/rust/src/project/sync_fs_to_automerge.rs +++ b/rust/src/project/sync_fs_to_automerge.rs @@ -2,9 +2,9 @@ use std::{path::PathBuf, sync::Arc}; use futures::StreamExt; use tokio::{ - sync::Mutex, - task::{JoinHandle, JoinSet}, + select, sync::Mutex, task::{JoinHandle, JoinSet} }; +use tokio_util::sync::CancellationToken; use crate::{ fs::{file_utils::FileContent, file_utils::FileSystemEvent}, @@ -22,43 +22,55 @@ pub struct SyncFileSystemToAutomerge { // Stream is good though because I ***think*** we can poll with now_or_never pending_changes: Arc>>, branch_db: BranchDb, - handle: JoinHandle<()>, + token: CancellationToken, } impl Drop for SyncFileSystemToAutomerge { fn drop(&mut self) { - self.handle.abort(); + self.token.cancel(); } } impl SyncFileSystemToAutomerge { pub fn new(branch_db: BranchDb) -> Self { let pending_changes = Arc::new(Mutex::new(Vec::new())); + let token = CancellationToken::new(); + let pending_changes_clone = pending_changes.clone(); let branch_db_clone = branch_db.clone(); - let handle = tokio::spawn(async move { + let token_clone = token.clone(); + + // TODO (Lilith): stick this on a method on an Inner struct like the rest + tokio::spawn(async move { let changes = FileSystemWatcher::start_watching( branch_db_clone.get_project_dir().clone(), branch_db_clone.clone(), ) .await; tokio::pin!(changes); - while let Some(event) = changes.next().await { - let (path, content) = match event { - FileSystemEvent::FileCreated(path, content) => (path, content), - FileSystemEvent::FileModified(path, content) => (path, content), - FileSystemEvent::FileDeleted(path) => (path, FileContent::Deleted), - }; - pending_changes_clone - .lock() - .await - .push((branch_db_clone.localize_path(&path), content)); + + loop { + select! { + event = changes.next() => { + let Some(event) = event else { continue; }; + let (path, content) = match event { + FileSystemEvent::FileCreated(path, content) => (path, content), + FileSystemEvent::FileModified(path, content) => (path, content), + FileSystemEvent::FileDeleted(path) => (path, FileContent::Deleted), + }; + pending_changes_clone + .lock() + .await + .push((branch_db_clone.localize_path(&path), content)); + }, + _ = token_clone.cancelled() => { break; } + } } }); Self { pending_changes, - handle, + token, branch_db, } } @@ -75,8 +87,11 @@ impl SyncFileSystemToAutomerge { return; } + tracing::info!("There are {:?} pending changes, attempting to commit...", pending_changes.len()); + // If the checked-out ref is invalid, we can't commit to the current branch. if checked_out_ref.as_ref().is_none_or(|r| !r.is_valid()) { + tracing::warn!("Can't commit to the current ref {:?}, because it isn't valid.", checked_out_ref); return; } @@ -90,6 +105,7 @@ impl SyncFileSystemToAutomerge { ) .await; if let Some(new_ref) = new_ref { + tracing::info!("Successfully made a commit! {:?}", new_ref); pending_changes.clear(); *checked_out_ref = Some(new_ref); } else { @@ -97,6 +113,7 @@ impl SyncFileSystemToAutomerge { } } + // TODO (Lilith): We need to check in the files... make it happen. /// Make an initial commit of ALL files from the filesystem to automerge. pub async fn checkin(&self) { // Because we always change the checked out ref after committing, we need to lock this in write mode. From cb28b8563df6e9a8a5038f9b3eefa782254b997d Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Sat, 24 Jan 2026 04:45:29 -0800 Subject: [PATCH 03/49] remove old stuff --- rust/src/fs/file_system_driver.rs | 754 ------------ rust/src/fs/file_system_driver/tests.rs | 421 ------- rust/src/project/project.rs | 1396 ----------------------- rust/src/project/project_driver.rs | 1276 --------------------- 4 files changed, 3847 deletions(-) delete mode 100644 rust/src/fs/file_system_driver.rs delete mode 100644 rust/src/fs/file_system_driver/tests.rs delete mode 100644 rust/src/project/project.rs delete mode 100644 rust/src/project/project_driver.rs diff --git a/rust/src/fs/file_system_driver.rs b/rust/src/fs/file_system_driver.rs deleted file mode 100644 index 63f62bb1..00000000 --- a/rust/src/fs/file_system_driver.rs +++ /dev/null @@ -1,754 +0,0 @@ -use crate::helpers::utils::ToShortForm; -use core::str; -use std::path::Path; -use futures::Stream; -use futures::stream::FuturesUnordered; -use futures::{ - StreamExt, - channel::mpsc::{UnboundedReceiver, UnboundedSender}, -}; -use glob::Pattern; -use notify::RecommendedWatcher; -use notify::{Config, RecursiveMode}; -use notify_debouncer_mini::{DebouncedEvent, Debouncer, new_debouncer_opt}; -#[cfg(not(target_os = "windows"))] -use rlimit::{Resource, getrlimit, setrlimit}; -use tokio::sync::mpsc::{Receiver, channel}; -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; -use std::sync::atomic::Ordering; -use std::time::Duration as StdDuration; -use std::{path::PathBuf, sync::atomic::AtomicBool}; -use tokio::sync::Mutex; -use tokio::{ - task::JoinHandle, - time::{Duration, sleep}, -}; -use tracing::instrument; -use notify::{Event, RecommendedWatcher, Watcher}; - -use super::file_utils::{FileContent, calculate_file_hash, get_buffer_and_hash}; - -// #[cfg(test)] -// mod tests; - -// static const var for debounce time -const DEBOUNCE_TIME: u64 = 100; - -impl ToShortForm for FileSystemEvent { - fn to_short_form(&self) -> String { - let content_type = match self { - FileSystemEvent::FileCreated(_, content) => match content { - FileContent::Scene(_) => "scene", - FileContent::String(_) => "text", - FileContent::Binary(_) => "binary", - FileContent::Deleted => "deleted", - }, - FileSystemEvent::FileModified(_, content) => match content { - FileContent::Scene(_) => "scene", - FileContent::String(_) => "text", - FileContent::Binary(_) => "binary", - FileContent::Deleted => "deleted", - }, - _ => "deleted", - }; - match self { - FileSystemEvent::FileCreated(path, _) => { - format!("FileCreated({:?}, {})", path, content_type) - } - FileSystemEvent::FileModified(path_buf, _) => { - format!("FileModified({:?}, {})", path_buf, content_type) - } - FileSystemEvent::FileDeleted(path_buf) => { - format!("FileDeleted({:?}, {})", path_buf, content_type) - } - } - } -} - -impl ToShortForm for Vec { - fn to_short_form(&self) -> String { - self.iter() - .map(|e| e.to_short_form()) - .collect::>() - .join(", ") - } -} - -#[derive(Debug)] -pub enum FileSystemUpdateEvent { - FileSaved(PathBuf, FileContent), - FileDeleted(PathBuf), - Pause, - Resume, -} - -impl ToShortForm for FileSystemUpdateEvent { - fn to_short_form(&self) -> String { - let content_type = match self { - FileSystemUpdateEvent::FileSaved(_, content) => match content { - FileContent::Scene(_) => "scene", - FileContent::String(_) => "text", - FileContent::Binary(_) => "binary", - FileContent::Deleted => "deleted", - }, - FileSystemUpdateEvent::FileDeleted(_) => "deleted", - FileSystemUpdateEvent::Pause => "", - FileSystemUpdateEvent::Resume => "", - }; - - match self { - FileSystemUpdateEvent::FileSaved(path, _) => { - format!("FileSaved({:?} {})", path, content_type) - } - FileSystemUpdateEvent::FileDeleted(path) => { - format!("FileDeleted({:?} {})", path, content_type) - } - FileSystemUpdateEvent::Pause => "Pause".to_string(), - FileSystemUpdateEvent::Resume => "Resume".to_string(), - } - } -} - -impl ToShortForm for Vec { - fn to_short_form(&self) -> String { - self.iter() - .map(|e| e.to_short_form()) - .collect::>() - .join(", ") - } -} - -#[derive(Debug, Clone)] -pub struct FileSystemTask { - watch_path: PathBuf, - file_hashes: Arc>>, - ignore_globs: Vec, - watcher: Arc>>, - // atomic bool - paused: Arc, - found_ignored_paths: Arc>>, -} - -#[derive(Debug)] -pub struct FileSystemDriver { - task: FileSystemTask, - output_rx: UnboundedReceiver, - input_tx: UnboundedSender, - handle: JoinHandle<()>, - rt: Option, -} - -impl FileSystemTask { - // Check if a path should be ignored based on glob patterns - fn should_ignore(&self, path: &PathBuf) -> bool { - // TODO: We should check if it's a symlink or not, but right now it's sufficient to just check if it's outside of the watch path - // check if it's outside of the watch path - if path.is_symlink() { - return true; - } - if !path.starts_with(&self.watch_path) { - return true; - } - let path_str = path.to_string_lossy(); - self.ignore_globs - .iter() - .any(|pattern| pattern.matches(&path_str)) - } - - async fn initialize_file_hashes_recur(&self, watch_path: PathBuf) -> tokio::io::Result<()> { - let mut dir = tokio::fs::read_dir(watch_path).await?; - let mut sub_tasks = FuturesUnordered::new(); - while let Some(entry) = dir.next_entry().await? { - let path = entry.path(); - - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - let mut found_ignored_paths = self.found_ignored_paths.lock().await; - found_ignored_paths.insert(path.clone()); - continue; - } - - if path.is_file() { - if let Some(hash) = calculate_file_hash(&path) { - let mut file_hashes = self.file_hashes.lock().await; - file_hashes.insert(path, hash); - } - } else if path.is_dir() { - let path = path.clone(); - sub_tasks.push(self.initialize_file_hashes_recur(path)); - } - } - while let Some(_) = sub_tasks.next().await {} - Ok(()) - } - - // Initialize the hash map with existing files - async fn initialize_file_hashes(&self) { - { - self.file_hashes.lock().await.clear(); - self.found_ignored_paths.lock().await.clear(); - } - self.initialize_file_hashes_recur(self.watch_path.clone()); - } - - // Handle file creation and modification events - async fn handle_file_event( - &self, - path: PathBuf, - ) -> Result, notify::Error> { - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - return Ok(None); - } - if !path.exists() { - // If the file doesn't exist, we want to emit a deleted event - let mut file_hashes: tokio::sync::MutexGuard<'_, HashMap> = - self.file_hashes.lock().await; - if file_hashes.contains_key(&path) { - file_hashes.remove(&path); - return Ok(Some(FileSystemEvent::FileDeleted(path))); - } - return Ok(None); - } - - if path.is_file() { - let mut result = get_buffer_and_hash(&path); - if result.is_err() { - sleep(StdDuration::from_millis(DEBOUNCE_TIME)).await; - result = get_buffer_and_hash(&path); - } - if result.is_err() { - tracing::error!("failed to get file content {:?}", result); - return Err(notify::Error::new(notify::ErrorKind::Generic( - "Failed to get file content".to_string(), - ))); - } - let (content, new_hash) = result.unwrap(); - let mut file_hashes: tokio::sync::MutexGuard<'_, HashMap> = - self.file_hashes.lock().await; - if file_hashes.contains_key(&path) { - let old_hash = file_hashes.get(&path).unwrap(); - if old_hash != &new_hash { - tracing::trace!("file {:?} changed, hash {} -> {}", path, old_hash, new_hash); - file_hashes.insert(path.clone(), new_hash); - return Ok(Some(FileSystemEvent::FileModified( - path, - FileContent::from_buf(content), - ))); - } - } else { - // If the file is newly created, we want to emit a created event - tracing::trace!("file {:?} created, hash {}", path, new_hash); - file_hashes.insert(path.clone(), new_hash); - return Ok(Some(FileSystemEvent::FileCreated( - path, - FileContent::from_buf(content), - ))); - } - } - Ok(None) - } - - // handle syncs from patchwork - async fn handle_file_update( - &self, - path: PathBuf, - content: FileContent, - ) -> Result<(), notify::Error> { - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - return Ok(()); - } - - // Write the file content to disk - if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { - let mut file_hashes = self.file_hashes.lock().await; - file_hashes.insert(path.clone(), hash_str); - } else { - return Err(notify::Error::new(notify::ErrorKind::Generic( - "Failed to write file".to_string(), - ))); - } - Ok(()) - } - - pub async fn handle_delete_update(&self, path: PathBuf) -> Result<(), notify::Error> { - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - return Ok(()); - } - - // Delete the file from disk - if std::fs::remove_file(&path.canonicalize().unwrap()).is_ok() { - // Remove the hash from our tracking - let mut file_hashes = self.file_hashes.lock().await; - file_hashes.remove(&path); - } else { - return Err(notify::Error::new(notify::ErrorKind::Generic( - "Failed to delete file".to_string(), - ))); - } - Ok(()) - } - - // Scan for changes in the file system - async fn scan_for_additive_changes(&self, watch_path: &PathBuf) -> Vec { - let mut events = Vec::new(); - let entries = std::fs::read_dir(watch_path); - if entries.is_err() { - return events; - } - let entries = entries.unwrap(); - for entry in entries.flatten() { - let path = entry.path(); - // Skip if path matches any ignore pattern - if self.should_ignore(&path) { - continue; - } - - if path.is_file() { - let res = self.handle_file_event(path).await; - if let Ok(Some(ret)) = res { - events.push(ret); - } - } else if path.is_dir() { - // Use Box::pin for the recursive call to avoid infinitely sized future - let sub_events = Box::pin(self.scan_for_additive_changes(&path)).await; - events.extend(sub_events); - } - } - events - } - - async fn process_notify_events( - &mut self, - notify_event: Vec, - output_tx: &UnboundedSender, - ) { - for event in notify_event { - if self.found_ignored_paths.contains(&event.path) { - continue; - } - if self.should_ignore(&event.path) { - self.found_ignored_paths.insert(event.path); - continue; - } - let result = self.handle_file_event(event.path.clone()).await; - if let Ok(Some(ret)) = result { - output_tx.unbounded_send(ret).ok(); - } - } - } - - async fn main_loop( - &mut self, - notify_rx: &mut UnboundedReceiver, notify::Error>>, - input_rx: &mut UnboundedReceiver, - output_tx: &UnboundedSender, - ) { - let mut sym_links = HashMap::new(); - self.initialize_file_hashes().await; - self.stop_watching_paths(&self.found_ignored_paths).await; - // Process both file system events and update events - loop { - tokio::select! { - // Handle file system events - Some(notify_result) = notify_rx.next() => { - if let Ok(notify_event) = notify_result { - self.process_notify_events(notify_event, output_tx).await; - } - }, - // Handle update events - Some(event) = input_rx.next() => { - match event { - FileSystemUpdateEvent::FileSaved(path, content) => { - let result = self.handle_file_update(path, content).await; - if result.is_err() { - tracing::error!("failed to handle file update {:?}", result); - } - } - FileSystemUpdateEvent::FileDeleted(path) => { - let result = self.handle_delete_update(path).await; - if result.is_err() { - tracing::error!("failed to handle file delete {:?}", result); - } - } - FileSystemUpdateEvent::Pause => { - self.stop_watching_path(&self.watch_path).await; - self.pause(); - } - FileSystemUpdateEvent::Resume => { - self.start_watching_path(&self.watch_path).await; - self.stop_watching_paths(&self.found_ignored_paths).await; - self.resume(); - // let events = self.scan_for_changes(&mut sym_links).await; - // for event in events { - // output_tx.unbounded_send(event).ok(); - // } - } - } - }, - } - } - } - - async fn stop_watching_paths(&self, paths: &HashSet) { - let mut watcher = self.watcher.lock().await; - for path in paths.iter() { - let _ret = watcher.watcher().unwatch(path); - } - } - - async fn stop_watching_path(&self, path: &PathBuf) { - let _ = self.watcher.lock().await.watcher().unwatch(path); - } - - async fn start_watching_path(&self, path: &PathBuf) { - let _ = self - .watcher - .lock() - .await - .watcher() - .watch(path, RecursiveMode::Recursive); - } - - pub fn is_paused(&self) -> bool { - self.paused.load(Ordering::Relaxed) - } - - fn pause(&self) { - self.paused.store(true, Ordering::Relaxed); - } - - fn resume(&self) { - self.paused.store(false, Ordering::Relaxed); - } -} - -const MAX_OPEN_FILES: u64 = 100000000; - -impl FileSystemDriver { - fn increase_ulimit() { - #[cfg(not(target_os = "windows"))] - { - let mut new_soft_limit = MAX_OPEN_FILES; - let mut new_hard_limit = MAX_OPEN_FILES; - let previous_result = getrlimit(Resource::NOFILE); - if let Err(e) = previous_result { - tracing::error!("failed to get ulimit {:?}", e); - } else if let Ok((soft_limit, hard_limit)) = previous_result { - tracing::debug!("soft ulimit {:?}", soft_limit); - tracing::debug!("hard ulimit {:?}", hard_limit); - if hard_limit > MAX_OPEN_FILES { - new_hard_limit = hard_limit; - } - if soft_limit > MAX_OPEN_FILES { - new_soft_limit = soft_limit; - } - } - - if let Err(e) = setrlimit(Resource::NOFILE, new_soft_limit, new_hard_limit) { - tracing::error!("failed to set ulimit {:?}", e); - } - let result = getrlimit(Resource::NOFILE); - if let Err(e) = result { - tracing::error!("failed to set ulimit {:?}", e); - } else if let Ok((soft_limit, hard_limit)) = result { - if soft_limit < MAX_OPEN_FILES || hard_limit < MAX_OPEN_FILES { - tracing::error!( - "failed to set ulimit; soft ulimit {:?}, hard ulimit {:?}", - soft_limit, - hard_limit - ); - } - } - } - } - - fn spawn_with_runtime( - watch_path: PathBuf, - ignore_globs: Vec, - rt: Option, - ) -> Self { - // if macos, increase ulimit to 100000000 - Self::increase_ulimit(); - let (output_tx, output_rx) = futures::channel::mpsc::unbounded(); - let (input_tx, mut input_rx) = futures::channel::mpsc::unbounded(); - // Spawn the file system watcher in a separate task - let notify_config = Config::default() - .with_follow_symlinks(false) - .with_ignore_globs(ignore_globs.clone()); - let ignore_globs: Vec = ignore_globs - .into_iter() - .filter_map(|glob_str| Pattern::new(&glob_str).ok()) - .collect(); - let (notify_tx, mut notify_rx) = futures::channel::mpsc::unbounded(); - - let debouncer_config = notify_debouncer_mini::Config::default() - .with_timeout(Duration::from_millis(DEBOUNCE_TIME)) - .with_batch_mode(true) - .with_notify_config(notify_config); - - let mut debouncer = new_debouncer_opt::<_, RecommendedWatcher>( - debouncer_config, - move |event: Result, notify::Error>| { - notify_tx.unbounded_send(event).unwrap(); - }, - ) - .unwrap(); - debouncer - .watcher() - .watch(&watch_path, RecursiveMode::Recursive) - .unwrap(); - - let rt_handle = if rt.is_some() { - rt.as_ref().unwrap().handle().clone() - } else { - tokio::runtime::Handle::current() - }; - let task: FileSystemTask = FileSystemTask { - watch_path: watch_path.clone(), - file_hashes: Arc::new(Mutex::new(HashMap::new())), - ignore_globs: ignore_globs, - watcher: Arc::new(Mutex::new(debouncer)), - paused: Arc::new(AtomicBool::new(false)), - found_ignored_paths: Arc::new(Mutex::new(HashSet::new())), - }; - - let mut this_task = task.clone(); - - let handle = rt_handle.spawn(async move { - this_task - .main_loop(&mut notify_rx, &mut input_rx, &output_tx) - .await; - }); - Self { - task, - output_rx, - input_tx, - handle, - rt: rt, - } - } - - pub fn spawn(watch_path: PathBuf, ignore_globs: Vec) -> Self { - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(1) - .thread_name("FileSystemDriver: watcher thread") - .enable_all() - .build() - .unwrap(); - Self::spawn_with_runtime(watch_path, ignore_globs, Some(rt)) - } - - fn get_existing_parent_path(path: &PathBuf) -> Option { - let mut parent = Some(path.as_path()); - while parent.is_some() { - if parent.as_ref().unwrap().exists() { - break; - } - parent = parent.unwrap().parent(); - } - if parent.is_some() { - return Some(parent.unwrap().to_path_buf()); - } - None - } - - async fn pause_task(&self) { - self.input_tx - .unbounded_send(FileSystemUpdateEvent::Pause) - .ok(); - while !self.task.is_paused() { - sleep(Duration::from_millis(100)).await; - } - } - - fn pause_task_blocking(&self) { - self.input_tx - .unbounded_send(FileSystemUpdateEvent::Pause) - .ok(); - while !self.task.is_paused() { - std::thread::sleep(Duration::from_millis(100)); - } - } - - async fn resume_task(&self) { - self.input_tx - .unbounded_send(FileSystemUpdateEvent::Resume) - .ok(); - while self.task.is_paused() { - sleep(Duration::from_millis(100)).await; - } - } - - fn resume_task_blocking(&self) { - self.input_tx - .unbounded_send(FileSystemUpdateEvent::Resume) - .ok(); - while self.task.is_paused() { - std::thread::sleep(Duration::from_millis(100)); - } - } - - pub async fn batch_update(&self, updates: Vec) { - self.pause_task().await; - { - let mut file_hashes = self.task.file_hashes.lock().await; - for update in updates { - match update { - FileSystemUpdateEvent::FileSaved(path, content) => { - if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { - file_hashes.insert(path.clone(), hash_str); - } else { - tracing::error!("failed to write file {:?}", path); - } - } - FileSystemUpdateEvent::FileDeleted(path) => { - file_hashes.remove(&path); - } - FileSystemUpdateEvent::Pause => { - continue; - } - FileSystemUpdateEvent::Resume => { - continue; - } - } - } - } - self.resume_task().await; - } - - #[instrument(skip_all, level = tracing::Level::INFO)] - pub fn batch_update_blocking( - &self, - updates: Vec, - ) -> Vec { - tracing::debug!("# of updates: {:?}", updates.len()); - tracing::trace!("updates: [{}]", updates.to_short_form()); - self.pause_task_blocking(); - tracing::trace!("batch_update_blocking after pause"); - let mut events: Vec = Vec::new(); - { - let mut file_hashes = self.task.file_hashes.blocking_lock(); - for update in updates { - match update { - FileSystemUpdateEvent::FileSaved(path, content) => { - let new_hash_str = content.to_hash(); - let mut modified = false; - let mut created = false; - if let Some(old_hash) = file_hashes.get(&path) { - if old_hash != &new_hash_str { - modified = true; - } - } else { - created = true; - } - if modified || created { - if let Ok(hash_str) = FileContent::write_file_content(&path, &content) { - if new_hash_str != hash_str { - tracing::error!( - "THIS SHOULD NOT HAPPEN: file {:?} previous calced hash {:?} != written hash {:?}", - path, - new_hash_str, - hash_str - ); - } - if modified { - tracing::trace!( - "file {:?} changed, hash {} -> {}", - path, - file_hashes.get(&path).unwrap(), - new_hash_str - ); - events - .push(FileSystemEvent::FileModified(path.clone(), content)); - } else { - tracing::trace!( - "file {:?} created, hash {}", - path, - new_hash_str - ); - events - .push(FileSystemEvent::FileCreated(path.clone(), content)); - } - file_hashes.insert(path, hash_str); - } else { - tracing::error!("failed to write file {:?}", path); - } - } else { - tracing::debug!( - "file {:?} already exists with same hash {:?}", - path, - new_hash_str - ); - } - } - FileSystemUpdateEvent::FileDeleted(path) => { - let _ = std::fs::remove_file(&path); - if file_hashes.remove(&path).is_some() { - events.push(FileSystemEvent::FileDeleted(path)); - } - } - _ => { - continue; - } - } - } - } - tracing::trace!("batch_update_blocking done, before resume"); - self.resume_task_blocking(); - tracing::debug!( - "batch_update_blocking done, updated files: {:?}", - events.len() - ); - tracing::trace!("events: [{}]", events.to_short_form()); - events - } - - pub fn has_events_pending(&self) -> bool { - self.output_rx.size_hint().0 > 0 - } - - pub fn try_next(&mut self) -> Option { - let res: Result, futures::channel::mpsc::TryRecvError> = - self.output_rx.try_next(); - if res.is_err() { - return None; - } - res.unwrap() - } - - pub async fn next(&mut self) -> Option { - self.output_rx.next().await - } - - pub async fn next_timeout(&mut self, timeout: Duration) -> Option { - let res = tokio::time::timeout(timeout, self.output_rx.next()).await; - if res.is_err() { - return None; - } - res.unwrap() - } - - pub fn stop(&self) { - self.handle.abort(); - } - - pub fn get_all_files_blocking(&self) -> Vec<(PathBuf, FileContent)> { - let file_hashes = self.task.file_hashes.blocking_lock(); - file_hashes - .iter() - .filter_map(|(path, _hash)| { - if path.is_file() { - let content = std::fs::read(path); - if content.is_ok() { - return Some((path.clone(), FileContent::from_buf(content.unwrap()))); - } - } - None - }) - .collect() - } -} diff --git a/rust/src/fs/file_system_driver/tests.rs b/rust/src/fs/file_system_driver/tests.rs deleted file mode 100644 index d07bf6d0..00000000 --- a/rust/src/fs/file_system_driver/tests.rs +++ /dev/null @@ -1,421 +0,0 @@ -// use std::collections::HashSet; -// use std::fs::File; -// use std::io::Write; -// use std::path::{Path, PathBuf}; -// use std::time::Duration; -// use crate::fs::file_system_driver::{DEBOUNCE_TIME, FileSystemDriver, FileSystemEvent, FileSystemUpdateEvent}; -// use crate::fs::file_utils::FileContent; -// use tempfile::tempdir; -// use tokio::{time::{sleep}}; -// use crate::helpers::utils::ToShortForm; - -// const WAIT_TIME: u64 = DEBOUNCE_TIME * 2; - -// fn replace_res_prefix(watch_path: &PathBuf, path: &Path) -> PathBuf { -// if path.to_string_lossy().starts_with("res://") { -// return watch_path.join(path.to_string_lossy().replace("res://", "")); -// } -// path.to_path_buf() -// } -// // Helper function to normalize paths for comparison -// fn normalize_path(watch_path: &PathBuf, path: &Path) -> PathBuf { -// // On macOS, /var is a symlink to /private/var, so we need to resolve it -// // if it begins with res://, replace it with the watch_path -// let path = replace_res_prefix(watch_path, path); -// if cfg!(target_os = "macos") { -// // Try to canonicalize the path, which resolves symlinks -// if let Ok(canonical) = path.canonicalize() { -// return canonical; -// } -// } -// // If canonicalization fails or we're not on macOS, just return the path as is -// path -// } - -// #[tokio::test] -// async fn test_file_system_watcher() { -// // Create a temporary directory for testing -// let dir = tempdir().unwrap(); -// // if macos, add /private/ to the start of the path -// let dir_path = dir.path().to_path_buf(); - -// // Create the file system driver -// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec!["*.tmp".to_string()], None); - - -// // Give the watcher time to initialize -// sleep(Duration::from_millis(2000)).await; - -// // Create a test file -// let test_file = dir_path.join("test.txt"); -// { -// let mut file = File::create(&test_file).unwrap(); -// file.write_all(b"test content").unwrap(); -// file.sync_all().unwrap(); -// } - -// // Wait for the create event -// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { -// match event { -// FileSystemEvent::FileCreated(path, content) => { -// assert_eq!(path, test_file); -// assert_eq!(content, FileContent::String("test content".to_string())); -// } -// _ => panic!("Unexpected event"), -// } -// } - -// let test_file2 = dir_path.join("test2.txt"); -// { -// let mut file = File::create(&test_file2).unwrap(); -// file.write_all(b"test content").unwrap(); -// file.sync_all().unwrap(); -// } - -// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { -// match event { -// FileSystemEvent::FileCreated(path, content) => { -// assert_eq!(path, test_file2); -// assert_eq!(content, FileContent::String("test content".to_string())); -// } -// _ => panic!("Unexpected event"), -// } -// } - -// // Modify the file with different content -// { -// let mut file = File::options().write(true).open(&test_file).unwrap(); -// file.write_all(b"modified content").unwrap(); -// file.sync_all().unwrap(); -// } - -// // Wait for the modify event -// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { -// match event { -// FileSystemEvent::FileModified(path, content) => { -// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); -// assert_eq!(content, FileContent::String("modified content".to_string())); -// } -// _ => panic!("Unexpected event"), -// } -// } - -// // Modify the file with the same content (should not trigger an event) -// let mut file = File::options().write(true).open(&test_file).unwrap(); -// file.write_all(b"modified content").unwrap(); - -// // Delete the file -// std::fs::remove_file(&test_file).unwrap(); - -// // Wait for the delete event -// // have it timeout after 100ms -// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { -// match event { -// FileSystemEvent::FileDeleted(path) => { -// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); -// } -// _ => panic!("Unexpected event"), -// } -// } else { -// panic!("No event received"); -// } -// } - -// #[tokio::test] -// async fn test_ignore_globs() { -// // Create a temporary directory for testing -// let dir = tempdir().unwrap(); -// // if macos, add /private/ to the start of the path -// let dir_path = if cfg!(target_os = "macos") { -// let mut path = dir.path().to_path_buf(); -// let private = PathBuf::from("/private"); -// path = private.join(path); -// path -// } else { -// dir.path().to_path_buf() -// }; - -// // Create the file system driver with ignore globs -// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec!["*.tmp".to_string()], None); - -// // Give the watcher time to initialize -// sleep(Duration::from_millis(100)).await; - - -// // Create a test file that should not be ignored -// let test_file = dir_path.join("test.txt"); -// let mut file = File::create(&test_file).unwrap(); -// file.write_all(b"test content").unwrap(); - - - -// // Wait for the create event (should only be for the non-ignored file) -// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { -// match event { -// FileSystemEvent::FileCreated(path, content) => { -// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); -// assert_eq!(content, FileContent::String("test content".to_string())); -// } -// _ => panic!("Unexpected event"), -// } -// } else { -// panic!("No event received"); -// } -// // Create a test file that should be ignored -// let ignored_file = dir_path.join("test.tmp"); -// let mut file = File::create(&ignored_file).unwrap(); -// file.write_all(b"test content").unwrap(); - - -// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { -// panic!("Unexpected event {:?}", event.to_short_form()); -// } -// { -// // Modify the ignored file (should not trigger an event) -// let mut file = File::options().write(true).open(&ignored_file).unwrap(); -// file.write_all(b"modified content").unwrap(); -// } -// { -// // Modify the non-ignored file (should trigger an event) -// let mut file = File::options().write(true).open(&test_file).unwrap(); -// file.write_all(b"modified content").unwrap(); -// // close it -// file.sync_all().unwrap(); -// } -// // Wait for the modify event (should only be for the non-ignored file) -// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { -// match event { -// FileSystemEvent::FileModified(path, _) => { -// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); -// } -// _ => panic!("Unexpected event"), -// } -// } else { -// panic!("No event received"); -// } - -// // Delete both files -// std::fs::remove_file(&test_file).unwrap(); - -// // Wait for the delete event (should only be for the non-ignored file) -// if let Some(event) = driver.next_timeout(Duration::from_millis(WAIT_TIME)).await { -// match event { -// FileSystemEvent::FileDeleted(path) => { -// assert_eq!(normalize_path(&dir_path, &path), normalize_path(&dir_path, &test_file)); -// } -// _ => panic!("Unexpected event"), -// } -// } else { -// panic!("No event received"); -// } -// std::fs::remove_file(&ignored_file).unwrap(); - -// // try_next should return None -// assert!(driver.next_timeout(Duration::from_millis(WAIT_TIME)).await.is_none()); - -// } - -// #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -// async fn test_file_system_update_events() { -// // Create a temporary directory for testing -// let dir = tempdir().unwrap(); -// // if macos, add /private/ to the start of the path -// let dir_path = dir.path().to_path_buf(); - -// // Create the file system driver -// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec![], None); - -// // Give the watcher time to initialize -// sleep(Duration::from_millis(100)).await; -// let test_content = "test content"; -// let modified_content = "modified content"; -// // Create a file via update event -// let test_file = dir_path.join("test.txt"); -// let result = driver.save_file(test_file.clone(), FileContent::String(test_content.to_string())).await; -// assert!(result.is_ok()); -// // sleep -// sleep(Duration::from_millis(WAIT_TIME)).await; -// // check that there's no event for the file created -// assert!(driver.try_next().is_none()); - -// // check that the file exists and contains the test_content -// assert!(test_file.exists()); -// assert_eq!(std::fs::read_to_string(&test_file).unwrap(), test_content); -// // modify the file -// let result = driver.save_file(test_file.clone(), FileContent::String(modified_content.to_string())).await; -// assert!(result.is_ok()); -// sleep(Duration::from_millis(WAIT_TIME)).await; - -// // check that there's no event for the file modified -// assert!(driver.try_next().is_none()); -// // check that the file exists and contains the modified_content -// assert!(test_file.exists()); -// assert_eq!(std::fs::read_to_string(&test_file).unwrap(), modified_content); -// // delete the file -// let result = driver.delete_file(test_file.clone()).await; -// assert!(result.is_ok()); -// sleep(Duration::from_millis(WAIT_TIME)).await; -// // check that there's no event for the file modified -// assert!(driver.try_next().is_none()); - -// // check that the file does not exist -// assert!(!test_file.exists()); -// } - - -// #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -// async fn test_file_system_large_number_of_files() { -// // we want to have at least 1000 files in the directory to test that we actually do raise the ulimit -// let dir = tempdir().unwrap(); -// let dir_path = dir.path().to_path_buf(); -// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec!["*.tmp".to_string()], None); -// // give the watcher time to initialize -// sleep(Duration::from_millis(100)).await; -// // create 1000 files -// let mut test_paths = HashSet::new(); -// for i in 0..1000 { -// let test_path = dir_path.join(format!("test_{}.txt", i)); -// let mut file = File::create(&test_path).unwrap(); -// file.write_all(b"test content").unwrap(); -// file.sync_all().unwrap(); -// test_paths.insert(test_path); -// } -// // wait for the watcher to process the events -// sleep(Duration::from_millis(100)).await; -// // check that the files exist -// // for i in 0..1000 { -// // let file = dir_path.join(format!("test_{}.txt", i)); -// // assert!(file.exists()); -// // } -// // now check to see if we have 1000 events -// let mut found_paths = HashSet::new(); -// while let Some(event) = driver.next_timeout(Duration::from_millis(10000)).await { -// let event_path = if let FileSystemEvent::FileCreated(path, _) = event { -// path -// } else if let FileSystemEvent::FileModified(path, _) = event { -// path -// } else { -// panic!("Unexpected event type {:?}", event.to_short_form()); -// }; -// found_paths.insert(event_path); -// if found_paths.len() == test_paths.len() { -// break; -// } -// } -// // assert_eq!(count, 1000); -// for path in test_paths { -// assert!(found_paths.contains(&path)); -// } -// } - -// #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -// async fn test_file_system_batch_update() { -// let dir = tempdir().unwrap(); -// let dir_path = dir.path().to_path_buf(); -// let mut driver = FileSystemDriver::spawn_with_runtime(dir_path.clone(), vec![], None); -// // update 1000 files -// let mut updates = Vec::new(); -// for i in 0..1000 { -// let test_path = dir_path.join(format!("test_{}.txt", i)); -// updates.push(FileSystemUpdateEvent::FileSaved(test_path, FileContent::String("test content".to_string()))); -// } -// driver.batch_update(updates).await; -// // wait for the watcher to process the events -// sleep(Duration::from_millis(100)).await; -// // check that the files exist -// { -// let hashes = driver.task.file_hashes.clone(); -// let hash_table = hashes.lock().await; -// for i in 0..1000 { -// let file = dir_path.join(format!("test_{}.txt", i)); -// assert!(file.exists()); -// assert!(hash_table.contains_key(&file)); -// } -// } -// // there should be no events emitted -// if let Some(event) = driver.next_timeout(Duration::from_millis(100)).await { -// assert!(false, "Unexpected event type {:?}", event.to_short_form()); -// } -// sleep(Duration::from_millis(1000)).await; -// // write a single file -// let test_path = dir_path.join("test_woop.txt"); -// let mut file = File::create(&test_path).unwrap(); -// file.write_all(b"test content").unwrap(); -// file.sync_all().unwrap(); -// sleep(Duration::from_millis(100)).await; -// // check that the file exists -// assert!(test_path.exists()); -// // check that we got an event for it -// if let Some(event) = driver.next().await { -// match event { -// FileSystemEvent::FileCreated(path, content) => { -// assert_eq!(path, test_path); -// assert_eq!(content, FileContent::String("test content".to_string())); -// } -// _ => panic!("Unexpected event"), -// } -// } else { -// panic!("No event received"); -// } - -// } -// // test blocking file system update -// #[test] -// fn test_file_system_blocking_update() { -// // set the default tokio runtime to multi_thread -// let dir = tempdir().unwrap(); -// let dir_path = dir.path().to_path_buf(); -// // spawn with the default runtime -// let mut driver = FileSystemDriver::spawn(dir_path.clone(), vec![]); -// // update 1000 files -// let mut updates = Vec::new(); -// for i in 0..1000 { -// let test_path = dir_path.join(format!("test_{}.txt", i)); -// updates.push(FileSystemUpdateEvent::FileSaved(test_path, FileContent::String("test content".to_string()))); -// } -// driver.batch_update_blocking(updates); -// // wait for the watcher to process the events -// // check that the files exist -// { -// let hashes = driver.task.file_hashes.clone(); -// let hash_table = hashes.blocking_lock(); -// for i in 0..1000 { -// let file = dir_path.join(format!("test_{}.txt", i)); -// assert!(file.exists()); -// assert!(hash_table.contains_key(&file)); -// } -// } -// // there should be no events emitted -// if let Some(event) = driver.try_next() { -// assert!(false, "Unexpected event type {:?}", event.to_short_form()); -// } -// std::thread::sleep(Duration::from_millis(1000)); -// // write a single file -// let test_path = dir_path.join("test_woop.txt"); -// let mut file = File::create(&test_path).unwrap(); -// file.write_all(b"test content").unwrap(); -// file.sync_all().unwrap(); -// std::thread::sleep(Duration::from_millis(1000)); -// // check that the file exists -// assert!(test_path.exists()); -// // check that we got an event for it -// if let Some(event) = driver.try_next() { -// match event { -// FileSystemEvent::FileCreated(path, content) => { -// assert_eq!(path, test_path); -// assert_eq!(content, FileContent::String("test content".to_string())); -// } -// _ => panic!("Unexpected event"), -// } -// } else { -// panic!("No event received"); -// } -// } - -// // #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -// // // run the test_file_system_large_number_of_files test 100 times -// // async fn test_file_system_large_number_of_files_100() { -// // for _ in 0..100 { -// // test_file_system_large_number_of_files().await; -// // } -// // } diff --git a/rust/src/project/project.rs b/rust/src/project/project.rs deleted file mode 100644 index a4101a14..00000000 --- a/rust/src/project/project.rs +++ /dev/null @@ -1,1396 +0,0 @@ -use ::safer_ffi::prelude::*; -use automerge::{ - ChangeHash, ObjId, ObjType, ROOT, ReadDoc -}; -use samod::{DocHandle, DocumentId, ConnectionInfo}; -use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; -use tracing::instrument; -use std::{cell::RefCell, collections::HashSet}; -use std::path::{PathBuf}; -use std::time::SystemTime; -use std::{collections::HashMap, str::FromStr}; -use crate::diff::differ::{Differ, ProjectDiff}; -use crate::fs::file_system_driver::{FileSystemDriver, FileSystemEvent, FileSystemUpdateEvent}; -use crate::fs::file_utils::FileContent; -use crate::helpers::branch::BranchState; -use crate::helpers::doc_utils::SimpleDocReader; -use crate::helpers::utils::{CommitInfo, ToShortForm, get_automerge_doc_diff, get_changed_files_vec, summarize_changes}; -use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkConfigAccessor, PatchworkEditorAccessor}; -use crate::project::project_driver::{ConnectionThreadError, DocHandleType, ProjectDriver, InputEvent, OutputEvent}; -use crate::project::project_api::{BranchViewModel, ChangeViewModel, ProjectViewModel}; - -/// Represents the state of the currently checked out branch. -#[derive(Debug, Clone)] -pub(super) enum CheckedOutBranchState { - /// No branch is currently checked out. - NothingCheckedOut(Option), - /// A branch is currently being checked out. - CheckingOut(DocumentId, Option), - /// A branch is currently checked out. - CheckedOut(DocumentId, Option), -} - -/// Manages the state and operations of a Patchwork project within Godot. -/// Its API is exposed to GDScript via the GodotProject struct. -#[derive(Debug)] -pub struct Project { - doc_handles: HashMap, - pub(super) branch_states: HashMap, - pub(super) checked_out_branch_state: CheckedOutBranchState, - project_doc_id: Option, - new_project: bool, - should_update_godot: bool, - pub(super) just_checked_out_new_branch: bool, - last_synced: Option<(DocumentId, Vec)>, - driver: Option, - pub(super) driver_input_tx: UnboundedSender, - driver_output_rx: UnboundedReceiver, - pub(super) sync_server_connection_info: Option, - file_system_driver: Option, - project_dir: String, - is_started: bool, - initial_load: bool, - pub(super) history: Vec, - pub(super) changes: HashMap, - // use RefCell for interior cache mutability - pub(super) diff_cache: RefCell, Vec), ProjectDiff>>, - last_ingest: (SystemTime, i32), - ingest_requested: bool -} - -impl Default for Project { - fn default() -> Self { - // TODO: Move driver input tx and output rx to the GodotProjectImpl struct, like in FileSystemDriver - let (driver_input_tx, _) = futures::channel::mpsc::unbounded(); - let (_, driver_output_rx) = futures::channel::mpsc::unbounded(); - Self { - sync_server_connection_info: None, - doc_handles: HashMap::new(), - branch_states: HashMap::new(), - checked_out_branch_state: CheckedOutBranchState::NothingCheckedOut(None), - project_doc_id: None, - new_project: true, - should_update_godot: false, - just_checked_out_new_branch: false, - last_synced: None, - driver: None, - driver_input_tx, - driver_output_rx, - file_system_driver: None, - project_dir: "".to_string(), - is_started: false, - initial_load: true, - history: Vec::new(), - changes: HashMap::new(), - diff_cache: RefCell::new(HashMap::new()), - last_ingest: (SystemTime::UNIX_EPOCH, 0), - ingest_requested: false - } - } -} - -/// The default server URL used for syncing Patchwork projects. Can be overridden by user or project configuration. -const DEFAULT_SERVER_URL: &str = "24.199.97.236:8085"; - -/// Notifications that can be emitted via process and consumed by GodotProject, in order to trigger signals to GDScript. -pub enum GodotProjectSignal { - CheckedOutBranch, - ChangesIngested -} - -impl Project { - pub fn globalize_path(&self, path: &String) -> String { - // trim the project_dir from the front of the path - if path.starts_with("res://") { - let thing = PathBuf::from(self.project_dir.clone()).join(PathBuf::from(&path["res://".len()..].to_string())); - - #[cfg(not(target_os = "windows"))] - { - return thing.to_string_lossy().to_string().replace("\\", "/"); - } - #[cfg(target_os = "windows")] - { - return thing.to_string_lossy().to_string(); - } - } else { - path.to_string() - } - } - - // TODO: We need to test this on Windows - pub fn localize_path(&self, path: &String) -> String { - let path = path.replace("\\", "/"); - let project_dir = self.project_dir.replace("\\", "/"); - if path.starts_with(&project_dir) { - let thing = PathBuf::from("res://".to_string()).join(PathBuf::from(&path[project_dir.len()..].to_string())); - thing.to_string_lossy().to_string() - } else { - path.to_string() - } - } - - pub fn get_project_doc_id(&self) -> Option { - self.project_doc_id.clone() - } - - /// Expensive operation to ingest all branch changes from automerge into the project data. - /// Should be called when we think there are new changes to process. - fn ingest_changes(&mut self) { - let Some(branch_state) = self.get_checked_out_branch_state() else { - return; - }; - - tracing::info!("Ingesting changes..."); - - let last_acked_heads = self.sync_server_connection_info - .as_ref() - .and_then(|i| i.docs.get(&branch_state.doc_handle.document_id())) - .and_then(|p| p.last_acked_heads.as_ref()); - - let changes = branch_state.doc_handle.with_document(|d| - d.get_changes(&[]) - .to_vec() - .iter() - .map(|c| { - CommitInfo::from(c) - }) - .collect::>() - ); - - self.history.clear(); - self.changes.clear(); - - // Check to see what the most recent ingested commit is. - let mut synced_until_index = -1; - for (i, change) in changes.iter().enumerate() { - if last_acked_heads.as_ref().is_some_and(|f| f.contains(&change.hash)) { - synced_until_index = i as i32; - } - } - - // Consume changes into self.changes - for (i, mut change) in changes.into_iter().enumerate() { - self.history.push(change.hash); - // If we're after the most recent ingested commit, we're not synced! - change.synced = (i as i32) <= synced_until_index; - change.summary = self.get_change_summary(&change); - self.changes.insert(change.hash, change); - } - } - - fn get_change_summary(&self, change: &CommitInfo) -> String { - (|| { - let meta = change.metadata.as_ref(); - let author = meta?.username.as_ref()?; - - // merge commit - if let Some(merge_info) = &meta?.merge_metadata { - let merged_branch = &self.get_branch(&merge_info.merged_branch_id.clone())?.get_name(); - return Some(format!("↪ {author} merged {merged_branch} branch")); - } - - // revert commit - if let Some(revert_info) = &meta?.reverted_to { - let heads = revert_info.iter() - .map(|s| &s[..7]) - .collect::>().join(", "); - return Some(format!("↩ {author} reverted to {heads}")); - } - - // initial commit - if change.is_setup() { - return Some(format!("Initialized repository")); - } - - return Some(summarize_changes(&author, meta?.changed_files.as_ref()?)); - })().unwrap_or("Invalid data".to_string()) - } - - pub fn get_branch_name(&self, branch_id: &DocumentId) -> String { - self.branch_states.get(branch_id).map(|b| b.name.clone()).unwrap_or(branch_id.to_string()) - } - - #[instrument(skip_all, level = tracing::Level::INFO)] - pub fn merge_branch(&mut self, source_branch_doc_id: DocumentId, target_branch_doc_id: DocumentId) { - println!(""); - tracing::info!("******** MERGE BRANCH: {:?} into {:?}", - self.get_branch_name(&source_branch_doc_id), - self.get_branch_name(&target_branch_doc_id) - ); - println!(""); - - self.driver_input_tx - .unbounded_send(InputEvent::MergeBranch { - source_branch_doc_id: source_branch_doc_id, - target_branch_doc_id: target_branch_doc_id.clone(), - }) - .unwrap(); - - // setting previous branch to None so that we don't delete any files when we checkout the new branch - self.checked_out_branch_state = CheckedOutBranchState::CheckingOut(target_branch_doc_id, None); - } - - pub fn create_merge_preview_branch_between( - &mut self, - source_branch_doc_id: DocumentId, - target_branch_doc_id: DocumentId, - ) { - println!(""); - tracing::info!("******** CREATE MERGE PREVIEW BRANCH: {:?} into {:?}", - self.get_branch_name(&source_branch_doc_id), - self.get_branch_name(&target_branch_doc_id) - ); - println!(""); - - self.driver_input_tx - .unbounded_send(InputEvent::CreateMergePreviewBranch { - source_branch_doc_id, - target_branch_doc_id, - }) - .unwrap(); - } - - pub fn create_revert_preview_branch_for(&mut self, branch_doc_id: DocumentId, revert_to: Vec) { - println!(""); - tracing::info!("******** CREATE REVERT PREVIEW BRANCH: {:?} to {:?}", - self.get_branch_name(&branch_doc_id), - revert_to.to_short_form() - ); - println!(""); - let branch_state = self.get_checked_out_branch_state().unwrap(); - let heads = branch_state.doc_handle.with_document(|d| { - d.get_heads() - }); - let content = self.get_changed_file_content_between(Some(branch_state.doc_handle.document_id().clone()), branch_state.doc_handle.document_id().clone(), heads.clone(), revert_to.clone(), true); - let files = content.into_iter().map(|event| { - match event { - FileSystemEvent::FileCreated(path, content) => (path.to_string_lossy().to_string(), content), - FileSystemEvent::FileModified(path, content) => (path.to_string_lossy().to_string(), content), - FileSystemEvent::FileDeleted(path) => (path.to_string_lossy().to_string(), FileContent::Deleted), - } - }).collect::>(); - - - self.driver_input_tx - .unbounded_send(InputEvent::CreateRevertPreviewBranch { - branch_doc_id, - files, - revert_to, - }) - .unwrap(); - - } - - - pub fn delete_branch(&mut self, branch_doc_id: DocumentId) { - self.driver_input_tx - .unbounded_send(InputEvent::DeleteBranch { branch_doc_id }) - .unwrap(); - } - - pub fn get_descendent_document( - &self, - previous_branch_id: DocumentId, - current_doc_id: DocumentId, - previous_heads: Vec, - current_heads: Vec, - ) -> Option { - let branch_state = match self.branch_states.get(¤t_doc_id) { - Some(branch_state) => branch_state, - None => return None, - }; - if current_heads.len() == 0 { - panic!("_get_descendent_document: current_heads is empty"); - } - if previous_heads.len() == 0 { - panic!("_get_descendent_document: previous_heads is empty"); - } - - if branch_state.doc_handle.with_document(|d| { - d.get_obj_id_at(ROOT, "files", &previous_heads).is_some() && - d.get_obj_id_at(ROOT, "files", ¤t_heads).is_some() - }) { - return Some(current_doc_id); - } - // try it with the other doc_id - let other_branch_state = match self.branch_states.get(&previous_branch_id) { - Some(branch_state) => branch_state, - None => { - tracing::error!("previous branch id {} not found", previous_branch_id); - return None; - } - }; - if other_branch_state.doc_handle.with_document(|d| { - d.get_obj_id_at(ROOT, "files", &previous_heads).is_some() && - d.get_obj_id_at(ROOT, "files", ¤t_heads).is_some() - }) { - return Some(previous_branch_id); - } - - - None - - } - - pub fn is_started(&self) -> bool { - self.is_started - } - - pub fn revert_to_heads(&mut self, to_revert_to: Vec) { - let branch_state = self.get_checked_out_branch_state().unwrap(); - let heads = branch_state.doc_handle.with_document(|d| { - d.get_heads() - }); - let content = self.get_changed_file_content_between(Some(branch_state.doc_handle.document_id().clone()), branch_state.doc_handle.document_id().clone(), heads.clone(), to_revert_to.clone(), true); - let files = content.into_iter().map(|event| { - match event { - FileSystemEvent::FileCreated(path, content) => (path, content), - FileSystemEvent::FileModified(path, content) => (path, content), - FileSystemEvent::FileDeleted(path) => (path, FileContent::Deleted), - } - }).collect::>(); - self.sync_files_at(branch_state.doc_handle.clone(), files, Some(heads), Some(to_revert_to), false); - self.checked_out_branch_state = CheckedOutBranchState::CheckingOut(branch_state.doc_handle.document_id().clone(), None); - } - - // INTERNAL FUNCTIONS - /// Gets the current file content on the current branch @ the current synced heads that changed - /// between the previous branch @ the previous heads and the current branch @ the current heads - #[instrument(skip_all, level = tracing::Level::DEBUG)] - pub(crate) fn get_changed_file_content_between( - &self, - previous_branch_id: Option, - current_doc_id: DocumentId, - previous_heads: Vec, - current_heads: Vec, - force_slow_diff: bool, - ) -> Vec { - - let current_branch_state = match self.branch_states.get(¤t_doc_id) { - Some(branch_state) => branch_state, - None => return Vec::new(), - }; - - let curr_heads = if current_heads.len() == 0 { - tracing::warn!("current heads is empty, using synced heads"); - current_branch_state.synced_heads.clone() - } else { - current_heads - }; - if previous_heads.len() == 0 { - tracing::debug!("No previous heads, getting all files on current branch {:?} between {} and {}", current_branch_state.name, previous_heads.to_short_form(), curr_heads.to_short_form()); - let files = self.get_files_on_branch_at(current_branch_state, Some(&curr_heads), None); - return files.into_iter().map(|(path, content)| { - match content { - FileContent::Deleted => { - FileSystemEvent::FileDeleted(PathBuf::from(path)) - } - _ => { - FileSystemEvent::FileCreated(PathBuf::from(path), content) - } - } - }).collect::>(); - } - - let descendent_doc_id: Option = if let Some(previous_branch_id) = previous_branch_id.clone() { - if previous_branch_id == current_doc_id { - Some(current_doc_id.clone()) - } else { - self.get_descendent_document(previous_branch_id, current_doc_id.clone(), previous_heads.clone(), curr_heads.clone()) - } - } else { - Some(current_doc_id.clone()) - }; - if descendent_doc_id.is_none() || force_slow_diff { - // neither document is the descendent of the other, we can't do a fast diff, - // we need to do it the slow way; get the files from both docs - // TODO: Is there a fast way to do this? - let previous_branch_state = match self.branch_states.get(&previous_branch_id.unwrap()) { - Some(branch_state) => branch_state, - None => { - tracing::warn!("_get_changed_file_content_between: previous branch id not found"); - return Vec::new(); - }, - }; - tracing::debug!("No descendent doc id, doing slow diff between previous {:?} @ {} and current {:?} @ {}", previous_branch_state.name, previous_heads.to_short_form(), current_branch_state.name, curr_heads.to_short_form()); - - let previous_files = self.get_files_on_branch_at(previous_branch_state, Some(&previous_heads), None); - let current_files = self.get_files_on_branch_at(current_branch_state, Some(&curr_heads), None); - let mut events = Vec::new(); - for (path, _) in previous_files.iter() { - if !current_files.contains_key(path) { - events.push(FileSystemEvent::FileDeleted(PathBuf::from(path))); - } - } - for (path, content) in current_files { - match content { - FileContent::Deleted => { - events.push(FileSystemEvent::FileDeleted(PathBuf::from(path))); - continue - } - _ => {} - } - if !previous_files.contains_key(&path) { - events.push(FileSystemEvent::FileCreated(PathBuf::from(path), content)); - } else if &content != previous_files.get(&path).unwrap() { - events.push(FileSystemEvent::FileModified(PathBuf::from(path), content)); - } - } - return events; - } - let descendent_doc_id = descendent_doc_id.unwrap(); - let branch_state = match self.branch_states.get(&descendent_doc_id) { - Some(branch_state) => branch_state, - None => panic!("_get_changed_file_content_between: descendent doc id not found"), - }; - tracing::debug!("descendent branch: {:?}, getting changes between {:?} @ {} and {:?} @ {}", - branch_state.name, - if let Some(previous_branch_id) = previous_branch_id { - self.get_branch_name(&previous_branch_id) - } else { - self.get_branch_name(¤t_doc_id) - }, - previous_heads.to_short_form(), - self.get_branch_name(¤t_doc_id), - curr_heads.to_short_form() - ); - let (patches, old_file_set, curr_file_set) = - branch_state.doc_handle.with_document(|d| { - let old_files_id: Option = d.get_obj_id_at(ROOT, "files", &previous_heads); - let curr_files_id = d.get_obj_id_at(ROOT, "files", &curr_heads); - let old_file_set = if old_files_id.is_none(){ - HashSet::::new() - } else { - d.keys_at(&old_files_id.unwrap(), &previous_heads).into_iter().collect::>() - }; - let curr_file_set = if curr_files_id.is_none(){ - HashSet::::new() - } else { - d.keys_at(&curr_files_id.unwrap(), &curr_heads).into_iter().collect::>() - }; - let patches = get_automerge_doc_diff( - d, - &previous_heads, - &curr_heads, - ); - (patches, old_file_set, curr_file_set) - }); - - let deleted_files = old_file_set.difference(&curr_file_set).into_iter().cloned().collect::>(); - let added_files = curr_file_set.difference(&old_file_set).into_iter().cloned().collect::>(); - let mut modified_files = HashSet::new(); - - // log all patches - let changed_files = get_changed_files_vec(&patches); - for file in changed_files { - if added_files.contains(&file) || deleted_files.contains(&file) { - continue; - } - modified_files.insert(file); - } - let make_event = |path: String, content: FileContent| { - if added_files.contains(&path) { - match content { - FileContent::Deleted => { - FileSystemEvent::FileDeleted(PathBuf::from(path)) - } - _ => { - FileSystemEvent::FileCreated(PathBuf::from(path), content) - } - } - } else if deleted_files.contains(&path) { - FileSystemEvent::FileDeleted(PathBuf::from(path)) - } else if modified_files.contains(&path) { - match content { - FileContent::Deleted => { - FileSystemEvent::FileDeleted(PathBuf::from(path)) - } - _ => { - FileSystemEvent::FileModified(PathBuf::from(path), content) - } - } - } else { - tracing::debug!("file not found in added_files, deleted_files, or modified_files: {:?}", path); - FileSystemEvent::FileModified(PathBuf::from(path), content) - } - }; - let mut changed_file_events = Vec::new(); - - let mut linked_doc_ids = Vec::new(); - for path in deleted_files.iter() { - changed_file_events.push(FileSystemEvent::FileDeleted(PathBuf::from(path))); - } - - branch_state.doc_handle.with_document(|doc|{ - let files_obj_id: ObjId = doc.get_at(ROOT, "files", &curr_heads).unwrap().unwrap().1; - - for path in doc.keys_at(&files_obj_id, &curr_heads) { - if !added_files.contains(&path) && !modified_files.contains(&path) { - continue; - } - - let file_entry = match doc.get_at(&files_obj_id, &path, &curr_heads) { - Ok(Some((automerge::Value::Object(ObjType::Map), file_entry))) => file_entry, - _ => { - tracing::error!("failed to get file entry for {:?}", path); - continue; - } - }; - - match FileContent::hydrate_content_at(file_entry, &doc, &path, &curr_heads) { - Ok(content) => { - changed_file_events.push(make_event(path, content)); - }, - Err(res) => { - match res { - Ok(id) => { - linked_doc_ids.push((id, path)); - }, - Err(error_msg) => { - tracing::error!("error: {:?}", error_msg); - } - } - } - }; - } - }); - - for (doc_id, path) in linked_doc_ids { - let linked_file_content: Option = self.get_linked_file(&doc_id); - if let Some(file_content) = linked_file_content { - changed_file_events.push(make_event(path, file_content)); - } - } - - changed_file_events - } - - - pub(crate) fn get_files_at(&self, heads: Option<&Vec>, filters: Option<&HashSet>) -> HashMap { - match &self.checked_out_branch_state { - CheckedOutBranchState::CheckedOut(branch_doc_id, _) => { - let branch_state = match self.branch_states.get(&branch_doc_id) { - Some(branch_state) => branch_state, - None => { - tracing::error!("_get_files_at: branch doc id {:?} not found", branch_doc_id); - return HashMap::new(); - }, - }; - self.get_files_on_branch_at(branch_state, heads, filters) - } - _ => panic!("_get_files_at: no checked out branch"), - } - } - - fn get_linked_file(&self, doc_id: &DocumentId) -> Option { - self.doc_handles.get(&doc_id) - .map(|doc_handle| { - doc_handle.with_document(|d| match d.get(ROOT, "content") { - Ok(Some((value, _))) if value.is_bytes() => { - Some(FileContent::Binary(value.into_bytes().unwrap())) - } - Ok(Some((value, _))) if value.is_str() => { - Some(FileContent::String(value.into_string().unwrap())) - } - _ => { - None - } - }) - }).unwrap_or(None) - } - - #[instrument(skip_all, level = tracing::Level::DEBUG)] - pub(crate) fn get_files_on_branch_at(&self, branch_state: &BranchState, heads: Option<&Vec>, filters: Option<&HashSet>) -> HashMap { - - let mut files = HashMap::new(); - - let heads = match heads { - Some(heads) => heads.clone(), - None => branch_state.synced_heads.clone(), - }; - tracing::debug!("Getting files on branch {:?} at {}", branch_state.name, heads.to_short_form()); - let mut linked_doc_ids = Vec::new(); - let filtered_paths = if let Some(filters) = filters { - filters - } else { - &HashSet::new() - }; - - branch_state.doc_handle.with_document(|doc|{ - let files_obj_id: ObjId = doc.get_at(ROOT, "files", &heads).unwrap().unwrap().1; - for path in doc.keys_at(&files_obj_id, &heads) { - if filtered_paths.len() > 0 && !filtered_paths.contains(&path) { - continue; - } - let file_entry = match doc.get_at(&files_obj_id, &path, &heads) { - Ok(Some((automerge::Value::Object(ObjType::Map), file_entry))) => file_entry, - _ => panic!("failed to get file entry for {:?}", path), - }; - - match FileContent::hydrate_content_at(file_entry, &doc, &path, &heads) { - Ok(content) => { - files.insert(path, content); - }, - Err(res) => { - match res { - Ok(id) => { - linked_doc_ids.push((id, path)); - }, - Err(error_msg) => { - tracing::error!("error: {:?}", error_msg); - } - } - } - }; - } - }); - - for (doc_id, path) in linked_doc_ids { - let linked_file_content: Option = self.get_linked_file(&doc_id); - if let Some(file_content) = linked_file_content { - files.insert(path, file_content); - } else { - tracing::warn!("linked file {:?} not found", path); - } - } - - return files; - - // try to read file as scene - } - - - #[instrument(skip_all, level = tracing::Level::INFO)] - fn sync_files_at(&self, - branch_doc_handle: DocHandle, - files: Vec<(PathBuf, FileContent)>, /* Record */ - heads: Option>, - revert: Option>, - new_project: bool) - { - let filter = files.iter().map(|(path, _)| path.to_string_lossy().to_string()).collect::>(); - println!(""); - tracing::debug!("******** SYNC: branch {:?} at {:?}, num files: {}", - self.branch_states.get(&branch_doc_handle.document_id()).map(|b| b.name.clone()).unwrap_or("unknown".to_string()), - if let Some(heads) = heads.as_ref() { - heads.to_short_form() - } else { - "".to_string() - }, - files.len() - ); - println!(""); - tracing::trace!("files: [{}]", - files.iter().map(|(path, content)| - format!("{}: {}", path.to_string_lossy().to_string(), content.to_short_form()) - ).collect::>().join(", ")); - let stored_files = self.get_files_at(heads.as_ref(), Some(&filter)); - let files_len = files.len(); - let changed_files: Vec<(String, FileContent)> = files.into_iter().filter_map(|(path, content)| { - let path = path.to_string_lossy().to_string(); - let stored_content = stored_files.get(&path); - if let Some(stored_content) = stored_content { - if stored_content == &content { - return None; - } - } - Some((path, content)) - }).collect(); - tracing::debug!("syncing {}/{} files", changed_files.len(), files_len); - tracing::trace!("syncing actually changed files: [{}]", changed_files.iter().map(|(path, content)| - format!("{}: {}", path, content.to_short_form()) - ).collect::>().join(", ")); - if let Some(revert_heads) = revert { - let _ = self.driver_input_tx - .unbounded_send(InputEvent::RevertTo { - branch_doc_handle, - heads, - files: changed_files, - revert_to: revert_heads, - }); - } else if new_project { - let _ = self.driver_input_tx - .unbounded_send(InputEvent::InitialCheckin { - branch_doc_handle, - heads, - files: changed_files, - }); - } else { - let _ = self.driver_input_tx - .unbounded_send(InputEvent::SaveFiles { - branch_doc_handle, - heads, - files: changed_files, - }); - } - } - - pub fn get_checked_out_branch_state(&self) -> Option<&BranchState> { - match &self.checked_out_branch_state { - CheckedOutBranchState::CheckedOut(branch_doc_id, _) => - self.branch_states.get(&branch_doc_id), - _ => None - } - } - - pub fn get_cached_diff( - &self, - heads_before: Vec, - heads_after: Vec - ) -> ProjectDiff { - self.diff_cache.borrow_mut() - .entry((heads_before.clone(), heads_after.clone())) - .or_insert_with(|| - self.get_diff(heads_before, heads_after)) - .clone() - } - - pub fn clear_diff_cache(&self) { - self.diff_cache.borrow_mut().clear(); - } - - pub fn get_diff(&self, heads_before: Vec, heads_after: Vec) -> ProjectDiff { - let Some(branch_state) = self.get_checked_out_branch_state() else { - return ProjectDiff::default(); - }; - let differ = Differ::new(self, heads_after, heads_before, branch_state); - differ.get_diff() - } - - async fn start_driver(&mut self) { - if self.driver.is_some() { - return; - } - let (driver_input_tx, driver_input_rx) = futures::channel::mpsc::unbounded(); - let (driver_output_tx, driver_output_rx) = futures::channel::mpsc::unbounded(); - self.driver_input_tx = driver_input_tx; - self.driver_output_rx = driver_output_rx; - - let storage_folder_path = self.globalize_path(&"res://.patchwork".to_string()); - let mut server_url = PatchworkConfigAccessor::get_project_value("server_url", ""); - if server_url.is_empty() { - server_url = PatchworkConfigAccessor::get_user_value("server_url", ""); - if server_url.is_empty() { - server_url = DEFAULT_SERVER_URL.to_string(); - tracing::info!("Using default server url: {:?}", server_url); - } else { - tracing::info!("Using user override for server url: {:?}", server_url); - } - } else { - tracing::info!("Using project override for server url: {:?}", server_url); - } - - let mut driver: ProjectDriver = ProjectDriver::create(storage_folder_path, server_url).await; - let maybe_user_name: String = PatchworkConfigAccessor::get_user_value("user_name", ""); - driver.spawn( - driver_input_rx, - driver_output_tx, - self.project_doc_id.clone(), - if maybe_user_name == "" { - None - } else { - Some(maybe_user_name) - }, - ); - self.driver = Some(driver); - } - - fn start_file_system_driver(&mut self) { - let project_path: String = self.globalize_path(&"res://".to_string()); - let project_path = PathBuf::from(project_path); - - let mut ignore_globs = vec![ - "**/.DS_Store".to_string(), - "**/thumbs.db".to_string(), - "**/desktop.ini".to_string(), - "**/patchwork.cfg".to_string(), - "**/addons/patchwork*".to_string(), - "**/target/*".to_string(), - // "**/.godot".to_string(), - "**/.*".to_string(), - // "**/.patchwork*".to_string(), - // "**/.patchwork/**/*".to_string(), - // "res://addons/patchwork/**/*".to_string(), - ]; - let mut parse_gitignore = |dir: PathBuf, file: &str| { - let path = dir.join(file); - let gitignore_content = if let Ok(content) = std::fs::read_to_string(path) { - content - } else { - String::new() - }; - - for line in gitignore_content.lines() { - // trim any comments and whitespace - let line = line.trim().split('#').next().unwrap_or_default().trim(); - if line.is_empty() { - continue; - } - let mut new_line = if line.starts_with("/") { - line.to_string() - } else { - dir.join(line).to_string_lossy().to_string() - }; - let new_line = if new_line.ends_with("/") { - // just remove the trailing slash - new_line.pop(); - new_line - } else { - new_line - }; - ignore_globs.push(new_line); - } - }; - parse_gitignore(project_path.clone(), ".gitignore"); - parse_gitignore(project_path.clone(), ".patchworkignore"); - parse_gitignore(project_path.clone(), ".gdignore"); - - - self.file_system_driver = Some(FileSystemDriver::spawn(project_path, ignore_globs)); - } - - pub fn start(&mut self) { - let project_doc_id: String = PatchworkConfigAccessor::get_project_value("project_doc_id", ""); - let checked_out_branch_doc_id = PatchworkConfigAccessor::get_project_value("checked_out_branch_doc_id", ""); - tracing::info!("Starting GodotProject with project doc id: {:?}", if project_doc_id == "" { "" } else { &project_doc_id }); - self.should_update_godot = false; - self.just_checked_out_new_branch = false; - self.last_synced = None; - self.project_doc_id = match DocumentId::from_str(&project_doc_id) { - Ok(doc_id) => Some(doc_id), - Err(_e) => None, - }; - self.new_project = match self.project_doc_id.is_none() { - true => true, - false => false, - }; - - self.checked_out_branch_state = match DocumentId::from_str(&checked_out_branch_doc_id) { - Ok(doc_id) => CheckedOutBranchState::CheckingOut(doc_id, None), - Err(_) => CheckedOutBranchState::NothingCheckedOut(None), - }; - - tracing::debug!( - "initial checked out branch state: {:?}", - self.checked_out_branch_state - ); - - // Bad practice; figure out a way to propogate this await to the UI instead - futures::executor::block_on(self.start_driver()); - self.start_file_system_driver(); - self.is_started = true; - // get the project path - } - - fn stop_driver(&mut self) { - if let Some(mut driver) = self.driver.take() { - driver.teardown(); - } - } - - pub fn stop(&mut self) { - if !self.is_started { - return; - } - self.stop_driver(); - if let Some(driver) = self.file_system_driver.take() { - driver.stop(); - } - self.checked_out_branch_state = CheckedOutBranchState::NothingCheckedOut(None); - self.sync_server_connection_info = None; - self.project_doc_id = None; - self.doc_handles.clear(); - self.branch_states.clear(); - self.file_system_driver = None; - self.is_started = false; - } - - pub fn safe_to_update_godot(initial_load: bool) -> bool { - return !(EditorFilesystemAccessor::is_scanning() || - PatchworkEditorAccessor::is_editor_importing() || - PatchworkEditorAccessor::is_changing_scene() || - (!initial_load && PatchworkEditorAccessor::unsaved_files_open()) - ); - } - - - /// Syncs the local state of the patchwork project document(s) from the - /// current local state to the current state at the current branch @ the current synced heads - /// the current local state is defined by the given branch @ the given heads - /// - /// from_branch_id is the branch that the current local state is on - /// from_heads is the heads that the current local state is on - #[instrument(skip_all, level = tracing::Level::INFO)] - fn sync_patchwork_to_godot(&mut self, from_branch_id: Option, from_heads: Vec) -> Vec { - println!(""); - tracing::debug!("*** SYNC PATCHWORK TO GODOT"); - let current_branch_state = match self.get_checked_out_branch_state() { - Some(branch_state) => branch_state, - None => { - tracing::error!("!!!!!!!no checked out branch!!!!!!"); - return Vec::new(); - } - }; - let current_doc_id = current_branch_state.doc_handle.document_id(); - // TODO: Do we want synced heads or the current heads? - let current_heads = current_branch_state.synced_heads.clone(); - let previous_heads = if from_heads.len() > 0 { - from_heads - } else { - match &from_branch_id { - Some(branch_id) => { - match self.branch_states.get(branch_id) { - Some(branch_state) => { - tracing::warn!("no previous branch heads, using current branch heads on {:?}", branch_state.name); - // TODO: Do we want synced heads or the current heads? - branch_state.synced_heads.clone() - } - None => { - tracing::error!("NO PREVIOUS BRANCH STATE?!?!?! Getting all changes from start to current_heads"); - Vec::new() - } - } - } - None => { - tracing::info!("no previous branch id, getting all changes from start to current_heads"); - Vec::new() - } - } - }; - if current_doc_id == from_branch_id.as_ref().unwrap_or(¤t_doc_id) && current_heads == previous_heads { - tracing::debug!("heads are the same, no changes to sync"); - return Vec::new(); - } - tracing::debug!("syncing branch {:?} from {}{} to {}", current_branch_state.name, - if from_branch_id.as_ref().unwrap_or(¤t_doc_id) != current_doc_id { - format!("{} @ ", self.get_branch_name(from_branch_id.as_ref().unwrap())) - } else { - "".to_string() - }, previous_heads.to_short_form(), current_heads.to_short_form()); - let events = self.get_changed_file_content_between(from_branch_id, current_doc_id.clone(), previous_heads, current_heads, false); - println!(""); - - let mut updates = Vec::new(); - for event in events { - match event { - FileSystemEvent::FileDeleted(path) => { - updates.push(FileSystemUpdateEvent::FileDeleted(PathBuf::from(self.globalize_path(&path.to_string_lossy().to_string()).to_string()))); - } - FileSystemEvent::FileCreated(path, content) => { - updates.push(FileSystemUpdateEvent::FileSaved(PathBuf::from(self.globalize_path(&path.to_string_lossy().to_string()).to_string()), content)); - } - FileSystemEvent::FileModified(path, content) => { - updates.push(FileSystemUpdateEvent::FileSaved(PathBuf::from(self.globalize_path(&path.to_string_lossy().to_string()).to_string()), content)); - } - } - } - if updates.len() == 0 { - tracing::debug!("no updates to sync"); - return Vec::new(); - } - if let Some(driver) = &mut self.file_system_driver { - let events = driver.batch_update_blocking(updates); - return events; - } - Vec::new() - } - - fn sync_godot_to_patchwork(&mut self, new_project: bool) { - match self.get_checked_out_branch_state() { - Some(branch_state) => { - // syncing the filesystem to patchwork - // get_files_at returns patchwork stuff, we need to get the files from the filesystem - if let Some(driver) = &self.file_system_driver { - let mut files = driver.get_all_files_blocking().into_iter().map( - |(path, content)| { - (self.localize_path(&path.to_string_lossy().to_string()).to_string(), content) - } - ).collect::>(); - if new_project { - // Hack to prevent long reloads when opening a new project; we just resave all the scenes that need it - let before_size: usize = files.len(); - files = files.into_iter().filter_map( - |(path, content)|{ - if let FileContent::Scene(content) = content { - return Some((path, FileContent::Scene(content))); - } - Some((path, content)) - } - ).collect::>(); - let events: Vec = driver.batch_update_blocking(Vec::new()); - if before_size - files.len() != events.len() { - tracing::error!("**** THIS SHOULD NOT HAPPEN: resaved {} files, but expected {} files back", before_size - files.len(), events.len()); - files = driver.get_all_files_blocking().into_iter().map( - |(path, content)| { - (self.localize_path(&path.to_string_lossy().to_string()).to_string(), content) - } - ).collect::>(); - } else { - files.extend(events.into_iter().map(|event| { - match event { - FileSystemEvent::FileCreated(path, content) => (self.localize_path(&path.to_string_lossy().to_string()), content), - FileSystemEvent::FileModified(path, content) => (self.localize_path(&path.to_string_lossy().to_string()), content), - FileSystemEvent::FileDeleted(path) => (self.localize_path(&path.to_string_lossy().to_string()), FileContent::Deleted) - } - })); - } - } - self.sync_files_at( - branch_state.doc_handle.clone(), - files.into_iter().map(|(path, content)| (PathBuf::from(path), content)).collect::>(), - Some(branch_state.synced_heads.clone()), - None, true); - } - } - None => panic!("couldn't save files, no checked out branch"), - }; - } - - fn get_previous_branch_id(&self) -> Option { - match &self.checked_out_branch_state { - CheckedOutBranchState::NothingCheckedOut(prev_branch_id) => prev_branch_id.clone(), - CheckedOutBranchState::CheckingOut(_, prev_branch_id) => prev_branch_id.clone(), - CheckedOutBranchState::CheckedOut(_, prev_branch_id) => prev_branch_id.clone(), - } - } - - pub fn new(project_dir: String) -> Self { - Self { - project_dir, - ..Default::default() - } - } - - /// Request for a change ingestion to be dispatched. - fn request_ingestion(&mut self) { - self.ingest_requested = true; - } - - /// If able, ingest changes, clear the ingestion request, and return true. - /// Otherwise, return false. - fn try_ingest_changes(&mut self) -> bool { - // Do not try to ingest if we haven't requested. - if !self.ingest_requested { - return false; - } - let now = SystemTime::now(); - let Ok(last_diff) = now.duration_since(self.last_ingest.0) else { return false; }; - - // Impose an arbitrary cap on requests within a time period. - // This is so that immediate syncs -- such as those from a local server -- don't have to wait before getting synced. - // But it also prevents spam of like a hundred slowing down the ingestion. - if last_diff.as_millis() < 100 { - if self.last_ingest.1 >= 3 { - return false; - } - } - else { - // since we're past the duration with no other requests, the counter resets. - self.last_ingest = (now, 0); - } - self.ingest_changes(); - self.ingest_requested = false; - self.last_ingest.1 += 1; - true - } - - // TODO: this is a very long and complicated method. Ideally it could be factored out to be simpler. - #[instrument(target = "patchwork_rust_core::godot_project::inner_process", level = tracing::Level::DEBUG, skip_all)] - pub fn process(&mut self, _delta: f64) -> (Vec, Vec) { - let mut signals: Vec = Vec::new(); - if self.try_ingest_changes() { - signals.push(GodotProjectSignal::ChangesIngested); - } - - if let Some(driver) = &mut self.driver { - if let Some(error) = driver.connection_thread_get_last_error() { - match error { - ConnectionThreadError::ConnectionThreadDied(error) => { - tracing::error!("automerge repo driver connection thread died, respawning: {}", error); - if !driver.respawn_connection_thread() { - tracing::error!("automerge repo driver connection thread failed too many times, aborting"); - // TODO: make the GUI do something with this - self.request_ingestion(); - } - } - ConnectionThreadError::ConnectionThreadError(error) => { - tracing::error!("automerge repo driver connection thread error: {}", error); - } - } - } - } - - let mut branches_changed = false; - while let Ok(Some(event)) = self.driver_output_rx.try_next() { - match event { - OutputEvent::NewDocHandle { - doc_handle, - doc_handle_type, - } => { - if doc_handle_type == DocHandleType::Binary { - tracing::trace!( - "NewBinaryDocHandle !!!! {} {} changes", - doc_handle.document_id(), - doc_handle.with_document(|d| d.get_heads().len()) - ); - } - - self.doc_handles - .insert(doc_handle.document_id().clone(), doc_handle.clone()); - } - OutputEvent::BranchStateChanged { - branch_state: new_branch_state, - trigger_reload, - } => { - let new_branch_state_doc_handle = new_branch_state.doc_handle.clone(); - let new_branch_state_doc_id = new_branch_state_doc_handle.document_id(); - self.branch_states - .insert(new_branch_state_doc_id.clone(), new_branch_state); - - branches_changed = true; - let mut checking_out_new_branch = false; - - let (active_branch_state, prev_branch_info) = match &self.checked_out_branch_state { - CheckedOutBranchState::NothingCheckedOut(prev_branch_id) => { - // check out main branch if we haven't checked out anything yet - let cloned_prev_branch_id = prev_branch_id.clone(); - let branch_state = self.branch_states.get(&new_branch_state_doc_handle.document_id()).unwrap(); - if branch_state.is_main { - checking_out_new_branch = true; - - self.checked_out_branch_state = CheckedOutBranchState::CheckingOut( - branch_state.doc_handle.document_id().clone(), - prev_branch_id.clone(), - ); - (Some(branch_state), cloned_prev_branch_id) - } else { - // we're still waiting for the project to be fully synced - (None, None) - } - } - CheckedOutBranchState::CheckingOut(branch_doc_id, prev_branch_info) => { - checking_out_new_branch = true; - (self.branch_states.get(branch_doc_id), prev_branch_info.clone()) - } - CheckedOutBranchState::CheckedOut(branch_doc_id, prev_branch_info) => { - (self.branch_states.get(branch_doc_id), prev_branch_info.clone()) - } - }; - - // only trigger update if checked out branch is fully synced - if let Some(active_branch_state) = active_branch_state { - if active_branch_state.is_synced() { - if checking_out_new_branch { - tracing::info!( - "TRIGGER checked out new branch: {}", - active_branch_state.name - ); - - self.checked_out_branch_state = CheckedOutBranchState::CheckedOut( - active_branch_state.doc_handle.document_id().clone(), - prev_branch_info, - ); - - self.just_checked_out_new_branch = true; - } else { - self.should_update_godot = self.should_update_godot || (new_branch_state_doc_id == active_branch_state.doc_handle.document_id() && trigger_reload); - if !trigger_reload { - tracing::debug!("TRIGGER saved changes: {}", active_branch_state.name); - self.request_ingestion(); - } - } - } - } - } - OutputEvent::Initialized { project_doc_id } => { - self.project_doc_id = Some(project_doc_id); - } - - OutputEvent::CompletedCreateBranch { branch_doc_id } => { - // PLEASE NOTE: If we change the logic such that we don't check out a new branch when we create one, - // we need to change _create_branch to not populate the previous branch id - self.checked_out_branch_state = - CheckedOutBranchState::CheckingOut(branch_doc_id, self.get_previous_branch_id()); - } - - OutputEvent::PeerConnectionInfoChanged { - peer_connection_info, - } => { - // TODO(Samod): Remove this hack - let Some(peer_connection_info) = peer_connection_info else { - continue; - }; - let Some(current_info) = self.sync_server_connection_info else { - self.sync_server_connection_info = peer_connection_info.clone(); - continue; - }; - match self.sync_server_connection_info.as_mut() - { - None => { - self.sync_server_connection_info = Some(peer_connection_info.clone()); - } - - Some(sync_server_connection_info) => { - sync_server_connection_info.last_received = - peer_connection_info.last_received; - sync_server_connection_info.last_sent = peer_connection_info.last_sent; - - peer_connection_info - .docs - .iter() - .for_each(|(doc_id, doc_state)| { - let had_previously_heads = sync_server_connection_info - .docs - .get(doc_id) - .is_some_and(|doc_state| { - doc_state - .clone() - .last_acked_heads - .is_some_and(|heads| heads.len() > 0) - }); - - // don't overwrite the doc state if it had previously had heads - // but now doesn't have any heads - if had_previously_heads - && doc_state - .clone() - .last_acked_heads - .is_some_and(|heads| heads.len() == 0) - { - return; - } - - sync_server_connection_info - .docs - .insert(doc_id.clone(), doc_state.clone()); - }); - } - }; - self.request_ingestion(); - } - } - } - - if branches_changed { - self.request_ingestion(); - } - - let has_pending_updates = self.just_checked_out_new_branch || self.should_update_godot; - let fs_driver_has_pending_updates = self.file_system_driver.as_ref().map(|driver| driver.has_events_pending()).unwrap_or(false); - if !has_pending_updates && !fs_driver_has_pending_updates { - return (Vec::new(), signals); - } - if !Self::safe_to_update_godot(self.initial_load) { - if has_pending_updates { - tracing::info!("Pending changes, but not safe to update godot, skipping..."); - } - if fs_driver_has_pending_updates { - tracing::info!("Pending editor changes to sync, but not safe to update godot, skipping..."); - } - return (Vec::new(), signals); - } - let (has_branch_state, previous_branch_info) = match &self.checked_out_branch_state{ - CheckedOutBranchState::NothingCheckedOut(_) => (false, None), - CheckedOutBranchState::CheckingOut(_, _) => (false, None), - CheckedOutBranchState::CheckedOut(_, prev_branch_info) => (true, prev_branch_info.clone()), - }; - if !has_branch_state { - if has_pending_updates { - tracing::info!("Pending changes, but we're not checked out on a branch, skipping..."); - } - if fs_driver_has_pending_updates { - tracing::info!("Pending editor changes to sync, but we're not checked out on a branch, skipping..."); - } - return (Vec::new(), signals); - } - let mut updates = Vec::new(); - if self.just_checked_out_new_branch { - self.just_checked_out_new_branch = false; - self.should_update_godot = false; - self.initial_load = false; - let (branch_name, checked_out_branch_doc_id) = self.get_checked_out_branch_state().map(|branch_state| - (branch_state.name.clone(), branch_state.doc_handle.document_id().clone()) - ).unwrap(); - tracing::debug!("just checked out branch {:?}", branch_name); - - let (previous_branch_id, previous_branch_heads) = - if self.new_project { - (None, Vec::new()) - } else if previous_branch_info.is_some() { - let heads = self.branch_states.get(previous_branch_info.as_ref().unwrap()).map(|branch_state| branch_state.synced_heads.clone()).unwrap_or_default(); - (previous_branch_info, heads) - } else if self.last_synced.is_some() && self.get_checked_out_branch_state().unwrap().merge_info.is_none() && self.last_synced.as_ref().map(|(doc_id, _)| doc_id) == Some(&checked_out_branch_doc_id){ - // TODO: this doesn't handle the case where we're starting up the editor and we're syncing the current doc state to the editor, - // the last_synced heads will be empty. - // We need to think about how to handle this case; if changes happened while outside of the editor, we want to sync everything. - // setting the from branch id to None to ensure it doesn't just sync the current heads - self.last_synced.as_ref().map(|(_doc_id, synced_heads)| (None, synced_heads.clone())).unwrap_or_default() - } else { - (None, Vec::new()) - }; - - if self.new_project { - self.new_project = false; - self.sync_godot_to_patchwork(true); - } else { - // Sync from the previous branch @ synced_heads to the current branch @ synced_heads - updates = self.sync_patchwork_to_godot(previous_branch_id, previous_branch_heads); - } - self.last_synced = self.get_checked_out_branch_state().map(|branch_state| (branch_state.doc_handle.document_id().clone(), branch_state.synced_heads.clone())); - // NOTE: it is VERY important that we save the project config AFTER we sync, - // because this will trigger a file scan and then resave the current project files in the editor - PatchworkConfigAccessor::set_project_value("project_doc_id", &match &self.get_project_doc_id() { - Some(doc_id) => doc_id.to_string(), - None => "".to_string(), - }); - PatchworkConfigAccessor::set_project_value("checked_out_branch_doc_id", &checked_out_branch_doc_id.to_string()); - signals.push(GodotProjectSignal::CheckedOutBranch); - self.request_ingestion(); - } else if self.should_update_godot { - self.initial_load = false; - // * Sync from the current branch @ previously synced_heads to the current branch @ synced_heads - tracing::debug!("should update godot"); - self.should_update_godot = false; - let current_branch_id = self.get_checked_out_branch_state().unwrap().doc_handle.document_id().clone(); - let last_synced_heads = self.last_synced.as_ref().map(|(branch_id, synced_heads)| - if branch_id == ¤t_branch_id { - synced_heads.clone() - } else { - Vec::new() - } - ).unwrap_or_default(); - updates = self.sync_patchwork_to_godot(Some(current_branch_id), last_synced_heads); - self.last_synced = self.get_checked_out_branch_state().map(|branch_state| (branch_state.doc_handle.document_id().clone(), branch_state.synced_heads.clone())); - self.request_ingestion(); - } else if let Some(fs_driver) = self.file_system_driver.as_mut() { - let mut new_files = Vec::new(); - while let Some(event) = fs_driver.try_next() { - match event { - FileSystemEvent::FileCreated(path, content) => { - new_files.push((path, content)); - } - FileSystemEvent::FileModified(path, content) => { - new_files.push((path, content)); - } - FileSystemEvent::FileDeleted(path) => { - new_files.push((path, FileContent::Deleted)); - } - } - } - if new_files.len() > 0 { - let files: Vec<(PathBuf, FileContent)> = new_files.into_iter().map( - |(path, content)| { - tracing::debug!("godot editor updated file: {:?}", path); - (PathBuf::from(self.localize_path(&path.to_string_lossy().to_string()).to_string()), content) - } - ).collect::>(); - - // TODO: Ask Paul about this tomorrow - self.sync_files_at(self.get_checked_out_branch_state().unwrap().doc_handle.clone(), files, None, None, false); - self.request_ingestion(); - } - } - - (updates, signals) - } -} diff --git a/rust/src/project/project_driver.rs b/rust/src/project/project_driver.rs deleted file mode 100644 index 4c935003..00000000 --- a/rust/src/project/project_driver.rs +++ /dev/null @@ -1,1276 +0,0 @@ -use automerge::Automerge; -use samod::{ConcurrencyConfig, ConnDirection, ConnectionInfo, DocHandle, DocumentId, Repo, Stopped}; -use futures::stream::FuturesUnordered; -use futures::{FutureExt, Stream}; -use std::collections::HashSet; -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time; -use std::{collections::HashMap, str::FromStr}; -use tokio::task::JoinHandle; - -use crate::helpers::branch::{BinaryDocState, BranchState, BranchStateForkInfo, BranchStateMergeInfo, BranchStateRevertInfo}; -use crate::fs::file_utils::FileContent; -use crate::parser::godot_parser::GodotScene; -use crate::helpers::doc_utils::SimpleDocReader; -use crate::helpers::utils::ToShortForm; -use crate::helpers::utils::{ - ChangeType, ChangedFile, CommitMetadata, MergeMetadata, commit_with_attribution_and_timestamp, get_default_patch_log, heads_to_vec_string, print_branch_state -}; -use crate::{ - helpers::branch::{BranchesMetadataDoc, GodotProjectDoc, ForkInfo, MergeInfo, Branch}, - helpers::utils::get_linked_docs_of_branch, -}; -use automerge::{ - transaction::Transactable, ChangeHash, ObjType, ReadDoc, - ROOT, -}; -use autosurgeon::{hydrate, reconcile }; -use futures::{ - channel::mpsc::{UnboundedReceiver, UnboundedSender}, - StreamExt, -}; - -use tokio::{net::TcpStream, runtime::Runtime}; - -#[derive(Clone, Debug)] -pub enum InputEvent { - CreateBranch { - name: String, - source_branch_doc_id: DocumentId, - }, - - CreateMergePreviewBranch { - source_branch_doc_id: DocumentId, - target_branch_doc_id: DocumentId, - }, - - CreateRevertPreviewBranch { - branch_doc_id: DocumentId, - files: Vec<(String, FileContent)>, - revert_to: Vec, - }, - - MergeBranch { - source_branch_doc_id: DocumentId, - target_branch_doc_id: DocumentId, - }, - - DeleteBranch { - branch_doc_id: DocumentId, - }, - - SaveFiles { - branch_doc_handle: DocHandle, - heads: Option>, - files: Vec<(String, FileContent)>, - }, - - InitialCheckin { - branch_doc_handle: DocHandle, - heads: Option>, - files: Vec<(String, FileContent)>, - }, - - RevertTo { - branch_doc_handle: DocHandle, - heads: Option>, - files: Vec<(String, FileContent)>, - revert_to: Vec, - }, - - SetUserName { - name: String, - } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum DocHandleType { - Binary, - Unknown, -} - -#[derive(Debug, Clone)] -pub enum OutputEvent { - Initialized { - project_doc_id: DocumentId, - }, - NewDocHandle { - doc_handle: DocHandle, - doc_handle_type: DocHandleType, - }, - - BranchStateChanged { - branch_state: BranchState, - trigger_reload: bool, - }, - - CompletedCreateBranch { - branch_doc_id: DocumentId, - }, - - PeerConnectionInfoChanged { - peer_connection_info: Option, - }, -} - -enum SubscriptionMessage { - Changed { - doc_handle: DocHandle, - }, - Added { - doc_handle: DocHandle, - }, -} - - -impl BranchState { - pub fn is_synced(&self) -> bool { - self.synced_heads == self.doc_handle.with_document(|d| d.get_heads()) - } -} - -struct DriverState { - tx: UnboundedSender, - repo_handle: Repo, - - user_name: Option, - - main_branch_doc_handle: DocHandle, - branches_metadata_doc_handle: DocHandle, - - binary_doc_states: HashMap, - branch_states: HashMap, - - pending_branch_doc_ids: HashSet, - pending_binary_doc_ids: HashSet, - - requesting_binary_docs: FuturesUnordered< - Pin, Stopped>)> + Send>>, - >, - requesting_branch_docs: FuturesUnordered< - Pin, Stopped>)> + Send>>, - >, - - subscribed_doc_ids: HashSet, - all_doc_changes: futures::stream::SelectAll< - std::pin::Pin + Send>>, - >, - - // heads that the frontend has for each branch doc - heads_in_frontend: HashMap> -} - - -#[derive(Debug)] -pub struct ProjectDriver { - // todo: make these not pub - pub runtime: Runtime, - pub repo_handle: Repo, - pub spawned_thread: Option>, -} - -impl ProjectDriver { - pub async fn create(storage_folder_path: String, server_url: String) -> Self { - // TODO: ensure we make this work across the program. Initialize it only once, etc. - let runtime: Runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(1) - .enable_all() - .thread_name("GodotProjectDriver: worker thread") - .build() - .unwrap(); - - let _guard = runtime.enter(); - - // let storage = FsStorage::open(storage_folder_path).unwrap(); - - // let repo = Repo::new(None, Box::new(storage)); - // let repo_handle = repo.run(); - - let storage = samod::storage::TokioFilesystemStorage::new(storage_folder_path); - - let repo_handle = Repo::build_tokio() - .with_concurrency(ConcurrencyConfig::Threadpool(rayon::ThreadPoolBuilder::new().build().unwrap())) - .with_storage(storage) - .load() - .await; - - return Self { - runtime, - repo_handle, - spawned_thread: None - }; - } - - pub fn spawn( - &mut self, - rx: UnboundedReceiver, - tx: UnboundedSender, - branches_metadata_doc_id: Option, - user_name: Option, - ) { - // Spawn sync task for all doc handles - self.spawned_thread = - Some(self.spawn_driver_task(rx, tx, branches_metadata_doc_id, &user_name)); - } - - pub fn teardown(&mut self) { - if let Some(spawned_thread) = self.spawned_thread.take() { - spawned_thread.abort(); - } - } - - fn spawn_driver_task( - &self, - mut rx: UnboundedReceiver, - tx: UnboundedSender, - branches_metadata_doc_id: Option, - user_name: &Option, - ) -> JoinHandle<()> { - let repo_handle = self.repo_handle.clone(); - let user_name = user_name.clone(); - - // let filter = EnvFilter::new("info").add_directive("automerge_repo=info".parse().unwrap()); - // if let Err(e) = tracing_subscriber::registry() - // .with(tracing_subscriber::fmt::layer().with_writer(std::io::stdout)) - // .with(filter) - // .try_init() - // { - // tracing::error!("Failed to initialize tracing subscriber: {:?}", e); - // } else { - // tracing::info!("Tracing subscriber initialized"); - // } - - return self.runtime.spawn(async move { - let branches_metadata_doc_handle; - let main_branch_doc_handle; - - // First, try the handle load locally. If we succeed, we're fine. - if let Ok(handles) = init_project_doc_handles(&repo_handle, branches_metadata_doc_id.as_ref(), user_name.as_ref()).await { - branches_metadata_doc_handle = handles.branches_metadata_doc_handle; - main_branch_doc_handle = handles.main_branch_doc_handle; - } - - // TODO: Handle this better in the refactor. - // Here, we spin until the server's connected. If the server is never connected, this will spin forever!!!!!!! - // If the server becomes connected and we can't load the doc still, we panic. - else { - loop { - if server_connected.load(Ordering::Relaxed) { - if let Ok(handles) = init_project_doc_handles(&repo_handle, branches_metadata_doc_id.as_ref(), user_name.as_ref()).await { - branches_metadata_doc_handle = handles.branches_metadata_doc_handle; - main_branch_doc_handle = handles.main_branch_doc_handle; - break; - } - panic!("Could not load project document even while the server was connected. This probably means the server doesn't actually have the requested doc!"); - } - tokio::time::sleep(time::Duration::from_millis(10)).await; - } - } - - tx.unbounded_send(OutputEvent::Initialized { project_doc_id: branches_metadata_doc_handle.document_id().clone() }).unwrap(); - - let mut state = DriverState { - tx: tx.clone(), - repo_handle: repo_handle.clone(), - user_name: user_name.clone(), - main_branch_doc_handle: main_branch_doc_handle.clone(), - binary_doc_states: HashMap::new(), - branch_states: HashMap::new(), - branches_metadata_doc_handle, - pending_branch_doc_ids: HashSet::new(), - pending_binary_doc_ids: HashSet::new(), - requesting_binary_docs : FuturesUnordered::new(), - requesting_branch_docs: FuturesUnordered::new(), - subscribed_doc_ids: HashSet::new(), - all_doc_changes: futures::stream::SelectAll::new(), - heads_in_frontend: HashMap::new(), - }; - - state.update_branch_doc_state(state.main_branch_doc_handle.clone()); - state.subscribe_to_doc_handle(state.branches_metadata_doc_handle.clone()); - state.subscribe_to_doc_handle(state.main_branch_doc_handle.clone()); - - let (_, peer_stream) = repo_handle.connected_peers(); - let mut peer_stream = peer_stream.fuse(); - loop { - futures::select! { - next = peer_stream.next() => { - if let Some(info) = next { - // TODO: do we need to update the synced_heads here? - // TODO: Currently, we only handle the very first peer connection here. - // We're making an explicit assumption that there's only ever 1 or 0 peers (the server.) - // Realistically this becomes a problem if there's other peers, but currently that's never the case. - tx.unbounded_send(OutputEvent::PeerConnectionInfoChanged { peer_connection_info: info.first().cloned() }).unwrap(); - }; - }, - - next = state.requesting_binary_docs.next() => { - if let Some((path, result)) = next { - match result { - Ok(Some(doc_handle)) => { - state.add_binary_doc_handle(&path, &doc_handle); - }, - Ok(None) => tracing::error!("binary doc not found"), - Err(_) => { - tracing::error!("error requesting binary doc: repo stopped"); - } - } - } - }, - - next = state.requesting_branch_docs.next() => { - if let Some((branch_name, result)) = next { - match result { - Ok(Some(doc_handle)) => { - state.pending_branch_doc_ids.remove(&doc_handle.document_id()); - state.update_branch_doc_state(doc_handle.clone()); - state.subscribe_to_doc_handle(doc_handle.clone()); - tracing::debug!("added branch doc: {:?}", branch_name); - - } - Ok(None) => tracing::error!("branch doc not found"), - Err(_) => { - tracing::error!("error requesting branch doc: repo stopped"); - } - } - } - }, - - message = state.all_doc_changes.select_next_some() => { - let doc_handle = match message { - SubscriptionMessage::Changed { doc_handle } => { - doc_handle - }, - SubscriptionMessage::Added { doc_handle } => { - tx.unbounded_send(OutputEvent::NewDocHandle { doc_handle: doc_handle.clone(), doc_handle_type: DocHandleType::Unknown }).unwrap(); - doc_handle - }, - }; - - let document_id = doc_handle.document_id(); - - // branches metadata doc changed - if document_id == state.branches_metadata_doc_handle.document_id() { - let branches = state.get_branches_metadata().branches.clone(); - - // check if there are new branches that haven't loaded yet - for (branch_id_str, branch) in branches.iter() { - let branch_id = DocumentId::from_str(branch_id_str).unwrap(); - let branch_name = branch.name.clone(); - - if !state.branch_states.contains_key(&branch_id) && !state.pending_branch_doc_ids.contains(&branch_id) { - state.pending_branch_doc_ids.insert(branch_id.clone()); - - state.requesting_branch_docs.push(repo_handle.find(branch_id.clone()).map(|doc_handle| { - (branch_name, doc_handle) - }).boxed()); - } - } - } - - // branch doc changed - if state.branch_states.contains_key(&document_id) { - state.update_branch_doc_state(doc_handle.clone()); - } - }, - - message = rx.select_next_some() => { - - match message { - InputEvent::CreateBranch { name, source_branch_doc_id } => { - state.create_branch(name.clone(), source_branch_doc_id.clone()).await; - }, - - InputEvent::CreateRevertPreviewBranch { branch_doc_id, files, revert_to } => { - state.create_revert_preview_branch(branch_doc_id, files, revert_to).await; - }, - - InputEvent::CreateMergePreviewBranch { source_branch_doc_id, target_branch_doc_id } => { - state.create_merge_preview_branch(source_branch_doc_id, target_branch_doc_id).await; - }, - - InputEvent::DeleteBranch { branch_doc_id } => { - state.delete_branch(branch_doc_id); - }, - - InputEvent::MergeBranch { source_branch_doc_id, target_branch_doc_id } => { - state.merge_branch(source_branch_doc_id, target_branch_doc_id); - }, - - InputEvent::SaveFiles { branch_doc_handle, files, heads } => { - state.save_files(branch_doc_handle, files, heads, false, None).await; - }, - - InputEvent::InitialCheckin { branch_doc_handle, files, heads } => { - state.save_files(branch_doc_handle, files, heads, true, None).await; - }, - - InputEvent::RevertTo { branch_doc_handle, files, heads, revert_to } => { - state.save_files(branch_doc_handle, files, heads, false, Some(revert_to)).await; - }, - - InputEvent::SetUserName { name } => { - state.user_name = Some(name.clone()); - } - }; - } - } - } - }); - } -} - -struct ProjectDocHandles { - branches_metadata_doc_handle: DocHandle, - main_branch_doc_handle: DocHandle, -} - -enum InitError { - ProjectDocumentNotFound, - MainBranchDocumentNotFound -} - -async fn init_project_doc_handles( - repo_handle: &Repo, - branches_metadata_doc_id: Option<&DocumentId>, - user_name: Option<&String>, -) -> Result { - match branches_metadata_doc_id { - // load existing project - Some(doc_id) => { - tracing::debug!("loading existing project: {:?}", doc_id); - - let Ok(branches_metadata_doc_handle) = repo_handle.find(doc_id.clone()).await else { - panic!("Failed init; repo stopped!"); - }; - // This happens if both us and the connected server do not know about this document. - // Also, it happens when we're not connected to the server at all. - let Some(branches_metadata_doc_handle) = branches_metadata_doc_handle else { - tracing::debug!("existing project document not found"); - return Err(InitError::ProjectDocumentNotFound); - }; - - let branches_metadata: BranchesMetadataDoc = - branches_metadata_doc_handle.with_document(|d| { - hydrate(d).unwrap_or_else(|_| { - panic!("failed init, can't hydrate metadata doc"); - }) - }); - - let Ok(main_branch_doc_handle) = repo_handle - .find(DocumentId::from_str(&branches_metadata.main_doc_id).unwrap()) - .await else - { - panic!("Failed init; repo stopped!") - }; - let Some(main_branch_doc_handle) = main_branch_doc_handle else { - tracing::debug!("existing main branch document not found"); - return Err(InitError::MainBranchDocumentNotFound); - }; - - return Ok(ProjectDocHandles { - branches_metadata_doc_handle: branches_metadata_doc_handle.clone(), - main_branch_doc_handle: main_branch_doc_handle.clone(), - }); - } - - // create new project - None => { - tracing::debug!("creating new project"); - - // Create new main branch doc - let main_branch_doc_handle = repo_handle.create(Automerge::new()).await.unwrap(); - main_branch_doc_handle.with_document(|d| { - let mut tx = d.transaction(); - let _ = reconcile( - &mut tx, - GodotProjectDoc { - files: HashMap::new(), - state: HashMap::new(), - }, - ); - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: user_name.cloned(), - branch_id: Some(main_branch_doc_handle.document_id().clone()), - merge_metadata: None, - reverted_to: None, - changed_files: None, - is_setup: Some(true) - }, - ); - }); - - let main_branch_doc_id = main_branch_doc_handle.document_id().to_string(); - let main_branch_doc_id_clone = main_branch_doc_id.clone(); - let branches = HashMap::from([( - main_branch_doc_id, - Branch { - name: String::from("main"), - id: main_branch_doc_handle.document_id().to_string(), - fork_info: None, - merge_info: None, - created_by: user_name.cloned(), - merged_into: None, - reverted_to: None, - }, - )]); - let branches_clone = branches.clone(); - - // create new branches metadata doc - let branches_metadata_doc_handle = repo_handle.create(Automerge::new()).await.unwrap(); - branches_metadata_doc_handle.with_document(|d| { - let mut tx = d.transaction(); - let _ = reconcile( - &mut tx, - BranchesMetadataDoc { - main_doc_id: main_branch_doc_id_clone, - branches: branches_clone, - }, - ); - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: user_name.cloned(), - branch_id: None, - merge_metadata: None, - reverted_to: None, - changed_files: None, - is_setup: Some(true) - }, - ); - }); - - return Ok(ProjectDocHandles { - branches_metadata_doc_handle: branches_metadata_doc_handle.clone(), - main_branch_doc_handle: main_branch_doc_handle.clone(), - }); - } - } -} - -impl DriverState { - async fn create_branch(&mut self, name: String, source_branch_doc_id: DocumentId) { - let source_branch_doc_handle = self - .branch_states - .get(&source_branch_doc_id) - .unwrap() - .doc_handle - .clone(); - - let new_branch_handle = clone_doc(&self.repo_handle, &source_branch_doc_handle).await; - - let branch = Branch { - name: name.clone(), - id: new_branch_handle.document_id().to_string(), - fork_info: Some(ForkInfo { - forked_from: source_branch_doc_id.to_string(), - forked_at: source_branch_doc_handle - .with_document(|d| d.get_heads()) - .iter() - .map(|h| h.to_string()) - .collect(), - }), - merge_info: None, - created_by: self.user_name.clone(), - merged_into: None, - reverted_to: None, - }; - - self.tx - .unbounded_send(OutputEvent::CompletedCreateBranch { - branch_doc_id: new_branch_handle.document_id().clone(), - }) - .unwrap(); - - self.branches_metadata_doc_handle.with_document(|d| { - let mut branches_metadata: BranchesMetadataDoc = hydrate(d).unwrap(); - let mut tx = d.transaction(); - branches_metadata.branches.insert(branch.id.clone(), branch); - let _ = reconcile(&mut tx, branches_metadata); - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: self.user_name.clone(), - branch_id: None, - merge_metadata: None, - reverted_to: None, - changed_files: None, - is_setup: Some(true) - }, - ); - }); - tracing::debug!("driver: created new branch: {:?}", new_branch_handle.document_id()); - } - - async fn create_revert_preview_branch( - &mut self, - branch_doc_id: DocumentId, - files: Vec<(String, FileContent)>, - revert_to: Vec, - ) { - tracing::debug!("driver: create revert preview branch"); - let branch_state = self.branch_states.get(&branch_doc_id).unwrap(); - let current_doc_id = branch_state.doc_handle.document_id(); - let current_heads = branch_state.doc_handle.with_document(|d| d.get_heads()); - // create a new branch doc, merge the original branch doc into it, and then commit the changes - let revert_preview_branch_doc_handle = clone_doc(&self.repo_handle, &branch_state.doc_handle).await; - let revert_preview_doc_id = revert_preview_branch_doc_handle.document_id(); - - let branch = Branch { - name: format!( - "{} <- {}", - revert_to.to_short_form(), current_heads.to_short_form() - ), - id: revert_preview_doc_id.to_string(), - fork_info: Some(ForkInfo { - forked_from: current_doc_id.to_string(), - forked_at: heads_to_vec_string(current_heads.clone()), - }), - merge_info: None, - created_by: self.user_name.clone(), - merged_into: None, - reverted_to: Some(revert_to.iter().map(|h| h.to_string()).collect()), - }; - self.branches_metadata_doc_handle.with_document(|d| { - let mut branches_metadata: BranchesMetadataDoc = hydrate(d).unwrap(); - let mut tx = d.transaction(); - branches_metadata.branches.insert(branch.id.clone(), branch); - let _ = reconcile(&mut tx, branches_metadata); - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: self.user_name.clone(), - branch_id: None, - merge_metadata: None, - reverted_to: None, - changed_files: None, - is_setup: Some(true) - }, - ); - }); - - self.save_files(revert_preview_branch_doc_handle.clone(), files, Some(current_heads), false, Some(revert_to)).await; - - self.tx - .unbounded_send(OutputEvent::CompletedCreateBranch { - branch_doc_id: revert_preview_doc_id.clone(), - }) - .unwrap(); - - } - - async fn create_merge_preview_branch( - &mut self, - source_branch_doc_id: DocumentId, - target_branch_doc_id: DocumentId, - ) { - tracing::debug!("driver: create merge preview branch"); - - let source_branch_state = self.branch_states.get(&source_branch_doc_id).unwrap(); - let target_branch_state = self.branch_states.get(&target_branch_doc_id).unwrap(); - - let merge_preview_branch_doc_handle = self.repo_handle.create(Automerge::new()).await.unwrap(); - - source_branch_state - .doc_handle - .with_document(|source_branch_doc| { - merge_preview_branch_doc_handle.with_document(|merge_preview_branch_doc| { - let _ = merge_preview_branch_doc.merge(source_branch_doc); - }); - }); - - target_branch_state - .doc_handle - .with_document(|target_branch_doc| { - merge_preview_branch_doc_handle.with_document(|merge_preview_branch_doc| { - let _ = merge_preview_branch_doc.merge(target_branch_doc); - }); - }); - - let branch = Branch { - name: format!( - "{} <- {}", - target_branch_state.name, source_branch_state.name - ), - id: merge_preview_branch_doc_handle.document_id().to_string(), - fork_info: Some(ForkInfo { - forked_from: source_branch_doc_id.to_string(), - forked_at: source_branch_state - .synced_heads - .iter() - .map(|h| h.to_string()) - .collect(), - }), - merge_info: Some(MergeInfo { - merge_into: target_branch_doc_id.to_string(), - merge_at: target_branch_state - .synced_heads - .iter() - .map(|h| h.to_string()) - .collect(), - }), - created_by: self.user_name.clone(), - merged_into: None, - reverted_to: None, - }; - - self.branches_metadata_doc_handle.with_document(|d| { - let mut branches_metadata: BranchesMetadataDoc = hydrate(d).unwrap(); - let mut tx = d.transaction(); - branches_metadata.branches.insert(branch.id.clone(), branch); - let _ = reconcile(&mut tx, branches_metadata); - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: self.user_name.clone(), - branch_id: Some(source_branch_doc_id), - merge_metadata: None, - reverted_to: None, - changed_files: None, - is_setup: Some(true) - }, - ); - }); - - self.tx - .unbounded_send(OutputEvent::CompletedCreateBranch { - branch_doc_id: merge_preview_branch_doc_handle.document_id().clone(), - }) - .unwrap(); - } - - // delete branch isn't fully implemented right now deletes are not propagated to the frontend - // right now this is just useful to clean up merge preview branches - fn delete_branch(&mut self, branch_doc_id: DocumentId) { - tracing::debug!("driver: delete branch {:?}", branch_doc_id); - - self.branches_metadata_doc_handle.with_document(|d| { - let mut tx = d.transaction(); - let mut branches_metadata: BranchesMetadataDoc = hydrate(&mut tx).unwrap(); - branches_metadata - .branches - .remove(&branch_doc_id.to_string()); - let _ = reconcile(&mut tx, branches_metadata); - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: self.user_name.clone(), - branch_id: None, - merge_metadata: None, - reverted_to: None, - changed_files: None, - is_setup: Some(true) - }, - ); - }); - } - - async fn save_files( - &mut self, - branch_doc_handle: DocHandle, - file_entries: Vec<(String, FileContent)>, - heads: Option>, - new_project: bool, - revert: Option> - ) { - let mut binary_entries: Vec<(String, DocHandle)> = Vec::new(); - let mut text_entries: Vec<(String, &String)> = Vec::new(); - let mut scene_entries: Vec<(String, &GodotScene)> = Vec::new(); - let mut deleted_entries: Vec = Vec::new(); - let is_revert = revert.is_some(); - - for (path, content) in file_entries.iter() { - match content { - FileContent::Binary(content) => { - let binary_doc_handle = self.repo_handle.create(Automerge::new()).await.unwrap(); - binary_doc_handle.with_document(|d| { - let mut tx = d.transaction(); - let _ = tx.put(ROOT, "content", content.clone()); - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: self.user_name.clone(), - branch_id: None, - merge_metadata: None, - reverted_to: None, - changed_files: None, - is_setup: Some(new_project) - }, - ); - }); - - self.add_binary_doc_handle(path, &binary_doc_handle); - binary_entries.push((path.clone(), binary_doc_handle)); - } - FileContent::String(content) => { - text_entries.push((path.clone(), content)); - } - FileContent::Scene(godot_scene) => { - scene_entries.push((path.clone(), godot_scene)); - } - FileContent::Deleted => { - deleted_entries.push(path.clone()); - } - } - } - branch_doc_handle.with_document(|d| { - let mut tx = match heads { - Some(heads) => d.transaction_at( - get_default_patch_log(), - &heads, - ), - None => d.transaction(), - }; - - let mut changes : Vec = Vec::new(); - let files = tx.get_obj_id(ROOT, "files").unwrap(); - - // write text entries to doc - for (path, content) in text_entries { - // get existing file url or create new one - let (file_entry, change_type) = match tx.get(&files, &path) { - Ok(Some((automerge::Value::Object(ObjType::Map), file_entry))) => (file_entry, ChangeType::Modified), - _ => (tx.put_object(&files, &path, ObjType::Map).unwrap(), ChangeType::Added) - }; - - changes.push(ChangedFile { - path, - change_type - }); - - // delete url in file entry if it previously had one - if let Ok(Some((_, _))) = tx.get(&file_entry, "url") { - let _ = tx.delete(&file_entry, "url"); - } - - // delete structured content in file entry if it previously had one - if let Ok(Some((_, _))) = tx.get(&file_entry, "structured_content") { - let _ = tx.delete(&file_entry, "structured_content"); - } - - // either get existing text or create new text - let content_key = match tx.get(&file_entry, "content") { - Ok(Some((automerge::Value::Object(ObjType::Text), content))) => content, - _ => tx - .put_object(&file_entry, "content", ObjType::Text) - .unwrap(), - }; - let _ = tx.update_text(&content_key, &content); - } - - // write scene entries to doc - for (path, godot_scene) in scene_entries { - // get the change flag - let change_type = match tx.get(&files, &path) { - Ok(Some(_)) => ChangeType::Modified, - _ => ChangeType::Added - }; - // godot_scene.reconcile(&mut tx, path.clone()); - let scene_file = tx.get_obj_id(&files, &path).unwrap_or_else(|| tx.put_object(&files, &path, ObjType::Map).unwrap()); - autosurgeon::reconcile_prop(&mut tx, &scene_file, "structured_content", godot_scene).unwrap_or_else(|e| { - tracing::error!("error reconciling scene: {}", e); - panic!("error reconciling scene: {}", e); - }); - changes.push(ChangedFile { - path, - change_type - }); - } - - // write binary entries to doc - for (path, binary_doc_handle) in binary_entries { - // get the change flag - let change_type = match tx.get(&files, &path) { - Ok(Some(_)) => ChangeType::Modified, - _ => ChangeType::Added - }; - - let file_entry = tx.put_object(&files, &path, ObjType::Map); - let _ = tx.put( - file_entry.unwrap(), - "url", - format!("automerge:{}", &binary_doc_handle.document_id()), - ); - - changes.push(ChangedFile { - path, - change_type - }); - } - - for path in deleted_entries { - let _ = tx.delete(&files, &path); - changes.push(ChangedFile { - path, - change_type: ChangeType::Removed - }); - } - - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: self.user_name.clone(), - branch_id: Some(branch_doc_handle.document_id().clone()), - merge_metadata: None, - reverted_to: match revert { - Some(revert) => Some(heads_to_vec_string(revert)), - None => None, - }, - changed_files: Some(changes), - is_setup: Some(new_project) - }, - ); - }); - - // update heads in frontend - if !new_project && !is_revert { - self.heads_in_frontend.insert( - branch_doc_handle.document_id().clone(), - branch_doc_handle.with_document(|d| d.get_heads()), - ); - } - - tracing::debug!("save on branch {:?} {:?}", branch_doc_handle.document_id(), self.heads_in_frontend); - } - - fn merge_branch(&mut self, source_branch_doc_id: DocumentId, target_branch_doc_id: DocumentId) { - let source_branch_state = self.branch_states.get(&source_branch_doc_id).unwrap(); - let target_branch_state = self.branch_states.get(&target_branch_doc_id).unwrap(); - - source_branch_state - .doc_handle - .with_document(|source_branch_doc| { - target_branch_state - .doc_handle - .with_document(|target_branch_doc| { - let _ = target_branch_doc.merge(source_branch_doc); - }); - }); - - let mut original_branch_id = None; - // if the branch has some merge_info we know that it's a merge preview branch - let merge_metadata = if source_branch_state.merge_info.is_some() { - let original_branch_state = self - .branch_states - .get(&source_branch_state.fork_info.as_ref().unwrap().forked_from) - .unwrap(); - - original_branch_id = Some(original_branch_state.doc_handle.document_id().to_string()); - - Some(MergeMetadata { - merged_branch_id: original_branch_state.doc_handle.document_id().clone(), - merged_at_heads: original_branch_state.synced_heads.clone(), - forked_at_heads: original_branch_state - .fork_info - .as_ref() - .unwrap() - .forked_at - .clone(), - }) - } else { - // todo: implement this case - None - }; - - if let Some(merge_metadata) = merge_metadata { - target_branch_state.doc_handle.with_document(|d| { - let mut tx = d.transaction(); - - // do a dummy change that we can attach some metadata to - let changed = tx.get_int(&ROOT, "_changed").unwrap_or(0); - let _ = tx.put(ROOT, "_changed", changed + 1); - - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: self.user_name.clone(), - branch_id: Some(target_branch_doc_id), - merge_metadata: Some(merge_metadata), - reverted_to: None, - changed_files: None, - is_setup: Some(false) - }, - ); - }); - let mut branch = self.get_branches_metadata() - .branches - .get(&source_branch_state.doc_handle.document_id().to_string()).unwrap().clone(); - branch.merged_into = original_branch_id; - self.branches_metadata_doc_handle.with_document(|d| { - let mut branches_metadata: BranchesMetadataDoc = hydrate(d).unwrap(); - let mut tx = d.transaction(); - branches_metadata.branches.insert(branch.id.clone(), branch); - let _ = reconcile(&mut tx, branches_metadata); - commit_with_attribution_and_timestamp( - tx, - &CommitMetadata { - username: self.user_name.clone(), - branch_id: Some(source_branch_doc_id), - merge_metadata: None, - reverted_to: None, - changed_files: None, - is_setup: Some(false) - }, - ); - }); - // self.branch_states.get_mut(&source_branch_doc_id).unwrap().merged_into = Some(target_branch_doc_id); - } - } - - fn update_branch_doc_state(&mut self, branch_doc_handle: DocHandle) { - let branch_state = match self.branch_states.get_mut(&branch_doc_handle.document_id()) { - Some(branch_state) => branch_state, - None => { - let branch = self - .get_branches_metadata() - .branches - .get(&branch_doc_handle.document_id().to_string()) - .unwrap() - .clone(); - - self.branch_states.insert( - branch_doc_handle.document_id().clone(), - BranchState { - name: branch.name.clone(), - doc_handle: branch_doc_handle.clone(), - linked_doc_ids: HashSet::new(), - synced_heads: Vec::new(), - fork_info: match branch.fork_info { - Some(fork_info) => Some(BranchStateForkInfo { - forked_from: DocumentId::from_str(&fork_info.forked_from).unwrap(), - forked_at: fork_info - .forked_at - .iter() - .map(|h| ChangeHash::from_str(h).unwrap()) - .collect(), - }), - None => None, - }, - merge_info: match branch.merge_info { - Some(merge_info) => Some(BranchStateMergeInfo { - merge_into: DocumentId::from_str(&merge_info.merge_into).unwrap(), - merge_at: merge_info - .merge_at - .iter() - .map(|h| ChangeHash::from_str(h).unwrap()) - .collect(), - }), - None => None, - }, - is_main: branch_doc_handle.document_id() - == self.main_branch_doc_handle.document_id(), - created_by: branch.created_by.clone(), - merged_into: match branch.merged_into { - Some(merged_into) => match DocumentId::from_str(&merged_into) { - Ok(merged_into) => Some(merged_into), - Err(_) => None, - }, - None => None, - }, - revert_info: match branch.reverted_to { - Some(reverted_to) => Some(BranchStateRevertInfo { - reverted_to: reverted_to.iter().map(|h| ChangeHash::from_str(h).unwrap()).collect(), - }), - None => None, - }, - }, - ); - self.branch_states - .get_mut(&branch_doc_handle.document_id()) - .unwrap() - } - }; - - let linked_docs = get_linked_docs_of_branch(&branch_doc_handle); - - // load binary docs if not already loaded - for (path, doc_id) in linked_docs.iter() { - if self.binary_doc_states.get(&doc_id).is_some() - || self.pending_binary_doc_ids.contains(&doc_id) - { - continue; - } - - self.pending_binary_doc_ids.insert(doc_id.clone()); - - let path = path.clone(); - self.requesting_binary_docs.push( - self.repo_handle - .find(doc_id.clone()) - .map(|doc_handle| (path, doc_handle)) - .boxed(), - ); - } - - // update linked doc ids - branch_state.linked_doc_ids = linked_docs.values().cloned().collect(); - - let missing_binary_doc_ids = - get_missing_binary_doc_ids(&branch_state, &self.binary_doc_states); - - // check if all linked docs have been loaded - if missing_binary_doc_ids.is_empty() { - branch_state.synced_heads = branch_doc_handle.with_document(|d| d.get_heads()); - - print_branch_state("branch doc state immediately loaded", &branch_state); - - self.tx - .unbounded_send(OutputEvent::BranchStateChanged { - branch_state: branch_state.clone(), - trigger_reload: !does_frontend_have_branch_at_heads( - &self.heads_in_frontend, - &branch_state - ), - }) - .unwrap(); - } - } - - fn add_binary_doc_handle(&mut self, _path: &String, binary_doc_handle: &DocHandle) { - self.binary_doc_states.insert( - binary_doc_handle.document_id().clone(), - BinaryDocState { - doc_handle: Some(binary_doc_handle.clone()), - }, - ); - - let _ = &self - .tx - .unbounded_send(OutputEvent::NewDocHandle { - doc_handle: binary_doc_handle.clone(), - doc_handle_type: DocHandleType::Binary, - }) - .unwrap(); - - // tracing::debug!("add_binary_doc_handle {:?} {:?}", path, binary_doc_handle.document_id()); - - // check all branch states that link to this doc - for branch_state in self.branch_states.values_mut() { - if branch_state - .linked_doc_ids - .contains(&binary_doc_handle.document_id()) - { - let missing_binary_doc_ids = - get_missing_binary_doc_ids(&branch_state, &self.binary_doc_states); - - // check if all linked docs have been loaded - if missing_binary_doc_ids.is_empty() { - branch_state.synced_heads = branch_state.doc_handle.with_document(|d| d.get_heads()); - self.tx - .unbounded_send(OutputEvent::BranchStateChanged { - branch_state: branch_state.clone(), - trigger_reload: !does_frontend_have_branch_at_heads( - &self.heads_in_frontend, - &branch_state - ), - }) - .unwrap(); - - tracing::debug!("branch {:?} (id: {:?}): state loaded with heads {}", branch_state.name, branch_state.doc_handle.document_id(), branch_state.synced_heads.to_short_form()); - } else { - tracing::debug!("branch {:?} (id: {:?}): state still missing {:?} binary docs", branch_state.name, branch_state.doc_handle.document_id(), missing_binary_doc_ids.len()); - tracing::trace!("missing binary doc ids: {:?}", missing_binary_doc_ids); - } - } - } - } - - pub fn subscribe_to_doc_handle(&mut self, doc_handle: DocHandle) { - if self.subscribed_doc_ids.contains(&doc_handle.document_id()) { - return; - } - - self.subscribed_doc_ids.insert(doc_handle.document_id().clone()); - self.all_doc_changes - .push(handle_changes(doc_handle.clone()).boxed()); - self.all_doc_changes.push( - futures::stream::once(async move { SubscriptionMessage::Added { doc_handle } }).boxed(), - ); - } - - fn get_branches_metadata(&self) -> BranchesMetadataDoc { - let branches_metadata: BranchesMetadataDoc = self - .branches_metadata_doc_handle - .with_document(|d| hydrate(d).unwrap()); - - return branches_metadata; - } -} - -async fn clone_doc(repo_handle: &Repo, doc_handle: &DocHandle) -> DocHandle { - let new_doc_handle = repo_handle.create(Automerge::new()).await.unwrap(); - - let _ = - doc_handle.with_document(|mut main_d| new_doc_handle.with_document(|d| d.merge(&mut main_d))); - - return new_doc_handle; -} - -fn handle_changes(handle: DocHandle) -> impl futures::Stream + Send { - futures::stream::unfold(handle, |doc_handle| async { - // There's currently a bug where removing this line causes changed() to not resolve the future (despite this line not actually doing anything). - // So, it'll spam with Changed events. - let _ = doc_handle.with_document(|d| d.get_heads()); - // TODO: this will probably break on upgrading automerge_repo because changed() is currently greedy, but will eventually check - // to see if there's an actual change before resolving the future. We rely on the greedy behavior here. - // let _ = doc_handle.changed().await; - - doc_handle.changes().next().await; - - Some(( - SubscriptionMessage::Changed { - doc_handle: doc_handle.clone(), - // diff, - }, - doc_handle, - )) - }) -} - -fn get_missing_binary_doc_ids( - branch_state: &BranchState, - binary_doc_states: &HashMap, -) -> Vec { - branch_state - .linked_doc_ids - .iter() - .filter(|doc_id| { - binary_doc_states - .get(doc_id) - .map_or(true, |binary_doc_state| { - binary_doc_state - .doc_handle - .as_ref() - .map_or(true, |handle| handle.with_document(|d| d.get_heads().is_empty())) - }) - }) - .cloned() - .collect::>() -} - -fn does_frontend_have_branch_at_heads( - heads_in_frontend: &HashMap>, - branch_state: &BranchState, -) -> bool { - tracing::trace!( - "Checking if frontend has branch {:?} (id: {:?}) at heads {:?}", - branch_state.name, - branch_state.doc_handle.document_id(), - heads_in_frontend - ); - - if let Some(synced_heads) = heads_in_frontend.get(&branch_state.doc_handle.document_id()) { - let result = synced_heads == &branch_state.synced_heads; - tracing::trace!("comparing {:?} == {:?}: {:?}", synced_heads, branch_state.synced_heads, result); - tracing::info!("Frontend has branch {:?} (id: {:?}) at heads {:?}: {:?}", branch_state.name, branch_state.doc_handle.document_id(), synced_heads, result); - result - } else { - tracing::info!("no synced heads found for branch {:?} (id: {:?})", branch_state.name, branch_state.doc_handle.document_id()); - false - } -} From 596d7a47aec4f3560c639347e036d03d6ec5884f Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:35:22 -0800 Subject: [PATCH 04/49] bunch more changes --- rust/src/helpers/tracing.rs | 11 +- rust/src/helpers/utils.rs | 10 +- rust/src/interop/godot_project.rs | 2 +- rust/src/project.rs | 6 +- rust/src/project/branch_db/branch.rs | 18 +- rust/src/project/branch_db/commit.rs | 10 + rust/src/project/branch_db/file.rs | 4 + rust/src/project/branch_db/util.rs | 8 + rust/src/project/change_ingester.rs | 235 +++++++++++ rust/src/project/document_watcher.rs | 13 +- rust/src/project/driver.rs | 317 +++++++------- rust/src/project/main_thread_block.rs | 79 ++++ rust/src/project/main_thread_block/test.rs | 42 ++ rust/src/project/peer_watcher.rs | 74 ++-- .../project/{new_project.rs => project.rs} | 199 +++------ rust/src/project/project_api_impl.rs | 85 ++-- rust/src/project/sync_automerge_to_fs.rs | 2 + rust/src/project/sync_fs_to_automerge.rs | 40 +- rust/tests/test.rs | 394 ------------------ 19 files changed, 788 insertions(+), 761 deletions(-) create mode 100644 rust/src/project/change_ingester.rs create mode 100644 rust/src/project/main_thread_block.rs create mode 100644 rust/src/project/main_thread_block/test.rs rename rust/src/project/{new_project.rs => project.rs} (56%) delete mode 100644 rust/tests/test.rs diff --git a/rust/src/helpers/tracing.rs b/rust/src/helpers/tracing.rs index 4df66689..b445cb62 100644 --- a/rust/src/helpers/tracing.rs +++ b/rust/src/helpers/tracing.rs @@ -3,7 +3,7 @@ use std::io::Write; use godot::classes::ProjectSettings; use tracing_appender::non_blocking::WorkerGuard; -use tracing_subscriber::{EnvFilter, Layer, fmt::{format::Writer, time::FormatTime}, layer::SubscriberExt, util::SubscriberInitExt}; +use tracing_subscriber::{EnvFilter, Layer, fmt::{format::{FmtSpan, Writer}, time::FormatTime}, layer::SubscriberExt, util::SubscriberInitExt}; use godot::obj::Singleton; fn get_user_dir() -> String { @@ -34,11 +34,12 @@ pub fn initialize_tracing() { let stdout_layer = tracing_subscriber::fmt::layer() .with_timer(CompactTime) .compact() + // .with_span_events(FmtSpan::ENTER | FmtSpan::CLOSE) .with_writer(CustomStdoutWriter::custom_stdout) - .with_filter(EnvFilter::new("info") - .add_directive("patchwork_rust_core=debug".parse().unwrap()) - .add_directive("samod=info".parse().unwrap()) - .add_directive("samod_core=info".parse().unwrap())); + .with_filter(EnvFilter::new("trace") + .add_directive("patchwork_rust_core=trace".parse().unwrap()) + .add_directive("samod=info".parse().unwrap()) + .add_directive("samod_core=info".parse().unwrap())); let file_layer = tracing_subscriber::fmt::layer() .with_line_number(true) .with_ansi(false) diff --git a/rust/src/helpers/utils.rs b/rust/src/helpers/utils.rs index f8e86198..76ac2bfa 100644 --- a/rust/src/helpers/utils.rs +++ b/rust/src/helpers/utils.rs @@ -65,14 +65,14 @@ pub(crate) fn parse_automerge_url(url: &str) -> Option { DocumentId::from_str(hash).ok() } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug)] pub struct MergeMetadata { pub merged_branch_id: DocumentId, pub merged_at_heads: Vec, pub forked_at_heads: Vec, } -#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] pub enum ChangeType { Added, Removed, @@ -89,13 +89,13 @@ impl fmt::Display for ChangeType { } } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug)] pub struct ChangedFile { pub change_type: ChangeType, pub path: String } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug)] pub struct CommitMetadata { pub username: Option, pub branch_id: Option, @@ -133,7 +133,7 @@ pub(crate) fn print_branch_state(message: &str, branch_state: &BranchState) { tracing::trace!("synced heads: {:?}", branch_state.synced_heads); } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct CommitInfo { pub hash: ChangeHash, pub timestamp: i64, diff --git a/rust/src/interop/godot_project.rs b/rust/src/interop/godot_project.rs index ce6b6994..f9719089 100644 --- a/rust/src/interop/godot_project.rs +++ b/rust/src/interop/godot_project.rs @@ -1,7 +1,7 @@ use crate::fs::file_utils::{FileContent, FileSystemEvent}; use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkConfigAccessor, PatchworkEditorAccessor}; use crate::project::driver::Driver; -use crate::project::new_project::{GodotProjectSignal, Project}; +use crate::project::project::{GodotProjectSignal, Project}; use crate::project::project_api::{BranchViewModel, ProjectViewModel}; use automerge::ChangeHash; use godot::classes::editor_plugin::DockSlot; diff --git a/rust/src/project.rs b/rust/src/project.rs index bbb50727..abf183e2 100644 --- a/rust/src/project.rs +++ b/rust/src/project.rs @@ -10,6 +10,8 @@ mod sync_automerge_to_fs; // pub for use in differ; consider restructuring pub mod branch_db; mod peer_watcher; -pub mod new_project; +pub mod project; // TODO (Lilith): Make this not pub -pub mod driver; \ No newline at end of file +pub mod driver; +pub mod main_thread_block; +mod change_ingester; \ No newline at end of file diff --git a/rust/src/project/branch_db/branch.rs b/rust/src/project/branch_db/branch.rs index 394cc1cd..8a3d79b8 100644 --- a/rust/src/project/branch_db/branch.rs +++ b/rust/src/project/branch_db/branch.rs @@ -9,22 +9,28 @@ use crate::{ branch::{Branch, BranchesMetadataDoc, GodotProjectDoc}, utils::{CommitMetadata, commit_with_attribution_and_timestamp}, }, - project::branch_db::BranchDb, + project::branch_db::{BranchDb, HistoryRef}, }; // Methods related to branch and document management on a [BranchDb]. impl BranchDb { /// Create a new metadata document, and a new main branch, and return the handle of the metadata document. + /// Checks out the initial commit of the main branch automatically. pub async fn create_metadata_doc(&self) -> DocHandle { + tracing::info!("Creating new metadata doc..."); let repo = self.inner.lock().await.repo.clone(); let username = self.inner.lock().await.username.clone(); + + // Because we always change the checked out ref after creating, we need to lock this in write mode. + let r = self.get_checked_out_ref_mut().await; + let mut checked_out_ref = r.write().await; // Create new main branch doc let main_handle = repo.create(Automerge::new()).await.unwrap(); let main_handle_clone = main_handle.clone(); let username_clone = username.clone(); - tokio::task::spawn_blocking(move || { + let new_heads = tokio::task::spawn_blocking(move || { main_handle_clone.with_document(|d| { let mut tx = d.transaction(); let _ = reconcile( @@ -45,10 +51,16 @@ impl BranchDb { is_setup: Some(true), }, ); - }); + d.get_heads() + }) }) .await .unwrap(); + + *checked_out_ref = Some(HistoryRef { + branch: main_handle.document_id().clone(), + heads: new_heads + }); let main_branch_doc_id = main_handle.document_id().to_string(); let main_branch_doc_id_clone = main_branch_doc_id.clone(); diff --git a/rust/src/project/branch_db/commit.rs b/rust/src/project/branch_db/commit.rs index d6d8e83f..56e91c4d 100644 --- a/rust/src/project/branch_db/commit.rs +++ b/rust/src/project/branch_db/commit.rs @@ -28,13 +28,21 @@ impl BranchDb { is_checking_in: bool, ) -> Option { let Some(branch_handle) = self.get_branch_handle(&ref_.branch).await else { + tracing::error!("Could not commit changes; ref doesn't have an associated branch handle! {:?}", ref_); return None; }; + tracing::info!("Attempting to commit changes..."); // Only commit files that have actually changed let files = self.filter_changed_files(ref_, files).await; + let count = files.len(); let username = self.inner.lock().await.username.clone(); + if (count == 0) { + tracing::info!("No actual changes found; not committing."); + return None; + } + let mut binary_entries: Vec<(String, DocHandle)> = Vec::new(); let mut text_entries: Vec<(String, String)> = Vec::new(); let mut scene_entries: Vec<(String, GodotScene)> = Vec::new(); @@ -174,6 +182,7 @@ impl BranchDb { .await .unwrap(); + tracing::info!("Committed {} files.", count); assert!(new_heads != ref_.heads); return Some(HistoryRef { heads: new_heads, @@ -217,6 +226,7 @@ impl BranchDb { pub async fn create_new_binary_doc(&self, content: Vec) -> DocHandle { + tracing::info!("Creating new binary doc..."); let handle = self .inner .lock() diff --git a/rust/src/project/branch_db/file.rs b/rust/src/project/branch_db/file.rs index d77cfc10..31cae0f1 100644 --- a/rust/src/project/branch_db/file.rs +++ b/rust/src/project/branch_db/file.rs @@ -61,12 +61,14 @@ impl BranchDb { /// Get a list of file operations between two points in Patchwork history. /// If one ref exists in the history of another, we can do a fast automerge diff. /// If they have diverged, we must do a slow file-wise diff. + #[tracing::instrument(skip_all)] pub async fn get_changed_file_content_between_refs( &self, old_ref: Option<&HistoryRef>, new_ref: &HistoryRef, force_slow_diff: bool, ) -> Option> { + tracing::info!("Getting changes between {:?} and {:?}", new_ref, old_ref); if !new_ref.is_valid() { tracing::warn!("new ref is empty, can't get changed files"); return None; @@ -211,11 +213,13 @@ impl BranchDb { .unwrap() } + #[tracing::instrument(skip_all)] pub async fn get_files_at_ref( &self, desired_ref: &HistoryRef, filters: &HashSet, ) -> Option> { + tracing::info!("Getting files at ref {:?}", desired_ref); let Some(branch_state) = self.get_branch_state(&desired_ref.branch).await else { return None; }; diff --git a/rust/src/project/branch_db/util.rs b/rust/src/project/branch_db/util.rs index bf3c5d28..05df17be 100644 --- a/rust/src/project/branch_db/util.rs +++ b/rust/src/project/branch_db/util.rs @@ -1,6 +1,7 @@ use std::{path::PathBuf, str::FromStr}; use samod::DocumentId; +use tracing::instrument; use crate::project::branch_db::{BranchDb, HistoryRef}; @@ -35,6 +36,7 @@ impl BranchDb { // or figure out a way to reliably update it when the heads actually change. // In the old system, synced heads was just force-updated every branch update. // Maybe that's enough? Get DocumentWatcher to do it? Then we remove the with_doc call here. + #[instrument(skip_all)] pub async fn get_latest_ref_on_branch(&self, branch: &DocumentId) -> Option { let state = self.get_branch_state(branch).await; let Some(state) = state else { @@ -76,4 +78,10 @@ impl BranchDb { .any(|pattern| pattern.matches(&path.to_string_lossy())) } + pub async fn get_branch_name(&self, id: &DocumentId) -> Option { + let Some(state) = self.get_branch_state(id).await else { + return None; + }; + Some(state.lock().await.name.clone()) + } } \ No newline at end of file diff --git a/rust/src/project/change_ingester.rs b/rust/src/project/change_ingester.rs new file mode 100644 index 00000000..49290c05 --- /dev/null +++ b/rust/src/project/change_ingester.rs @@ -0,0 +1,235 @@ +use std::{ + sync::Arc, + time::{Duration, SystemTime}, +}; + +use futures::StreamExt; +use tokio::{ + select, + sync::{Mutex, Notify, watch}, +}; +use tokio_util::sync::CancellationToken; + +use crate::{ + helpers::utils::{CommitInfo, CommitMetadata, summarize_changes}, + project::{branch_db::BranchDb, peer_watcher::PeerWatcher, project_api::ChangeViewModel}, +}; + +#[derive(Debug)] +pub struct ChangeIngester { + inner: Arc, + token: CancellationToken, +} + +#[derive(Debug)] +struct ChangeIngesterInner { + changes_tx: watch::Sender>, + ingestion_request: Notify, + peer_watcher: Arc, + token: CancellationToken, + last_ingest: Mutex<(SystemTime, i32)>, + branch_db: BranchDb, +} + +impl Drop for ChangeIngester { + fn drop(&mut self) { + self.token.cancel() + } +} + +impl ChangeIngester { + pub fn new(peer_watcher: Arc, branch_db: BranchDb) -> Self { + let (changes_tx, _) = watch::channel(Vec::new()); + let token = CancellationToken::new(); + let ingestion_request = Notify::new(); + let inner = Arc::new(ChangeIngesterInner { + changes_tx, + peer_watcher, + ingestion_request, + token: token.clone(), + last_ingest: Mutex::new((SystemTime::UNIX_EPOCH, 0)), + branch_db, + }); + + let inner_clone = inner.clone(); + tokio::spawn(async move { + let stream = inner_clone.peer_watcher.subscribe(); + tokio::pin!(stream); + + loop { + select! { + _ = inner_clone.token.cancelled() => { break; } + _ = stream.next() => { + inner_clone.ingestion_request.notify_one(); + } + // _ = inner_clone.ingestion_request.notified() => { + // inner_clone.ingest_changes().await; + // }, + } + } + }); + + Self { token, inner } + } + + pub fn request_ingestion(&self) { + self.inner.ingestion_request.notify_one(); + } + + // I don't like exposing this, but it's the simplest solution for now. + pub fn get_changes_rx(&self) -> watch::Receiver> { + self.inner.changes_tx.subscribe() + } +} + +impl ChangeIngesterInner { + async fn ingest_changes(&self) { + let mut last_ingest = self.last_ingest.lock().await; + let now = SystemTime::now(); + let last_diff = now + .duration_since(last_ingest.0) + .unwrap_or(Duration::from_secs(0)); + + // Impose an arbitrary cap on requests within a time period. + // This is so that immediate syncs -- such as those from a local server -- don't have to wait before getting synced. + // But it also prevents spam of like a hundred slowing down the thread. + let max_requests_before_debounce = 3; + let debounce = 100; + if last_diff.as_millis() < debounce { + if last_ingest.1 >= max_requests_before_debounce { + tokio::time::sleep(Duration::from_millis( + (debounce - last_diff.as_millis()) as u64, + )) + .await; + } + } else { + // since we're past the duration with no other requests, the counter resets. + *last_ingest = (now, 0); + } + self.changes_tx.send(self.get_changes().await).unwrap(); + last_ingest.1 += 1; + } + + async fn get_change_summary(&self, change: &CommitInfo) -> Option { + let meta = change.metadata.as_ref(); + let author = meta?.username.clone().unwrap_or("Anonymous".to_string()); + + // merge commit + if let Some(merge_info) = &meta?.merge_metadata { + let merged_branch = self + .branch_db + .get_branch_name(&merge_info.merged_branch_id.clone()) + .await + .unwrap_or(merge_info.merged_branch_id.to_string()); + return Some(format!("↪ {author} merged {merged_branch}")); + } + + // revert commit + if let Some(revert_info) = &meta?.reverted_to { + let heads = revert_info + .iter() + .map(|s| &s[..7]) + .collect::>() + .join(", "); + return Some(format!("↩ {author} reverted to {heads}")); + } + + // initial commit + if change.is_setup() { + return Some(format!("Initialized repository")); + } + + return Some(summarize_changes(&author, meta?.changed_files.as_ref()?)); + } + + /// Gets the changes from the current branch and returns it. + // TODO (Lilith): This is MISERABLY slow due to the with_document. + // Maybe figure out a way to factor that out. + #[tracing::instrument(skip_all)] + async fn get_changes(&self) -> Vec { + let checked_out = self.branch_db.get_checked_out_ref_mut().await; + let checked_out = checked_out.read().await; + let Some(checked_out) = checked_out.as_ref() else { + tracing::info!("Can't get changes; nothing checked out!"); + return Vec::new(); + }; + + let Some(branch_state) = self.branch_db.get_branch_state(&checked_out.branch).await else { + tracing::info!("Can't get the checked out branch state; something must be wrong"); + return Vec::new(); + }; + + // TODO: we probably don't need to lock the branch state for this whole method + let branch_state = branch_state.lock().await; + let handle = branch_state.doc_handle.clone(); + let doc_id = handle.document_id(); + + let last_acked_heads = self + .peer_watcher + .get_server_info() + .as_ref() + .and_then(|info| info.docs.get(&doc_id)) + .and_then(|state| state.last_acked_heads.clone()); + + let h = handle.clone(); + let changes = tokio::task::spawn_blocking(move || { + h.with_document(move |d| { + d.get_changes(&[]) + .to_vec() + .iter() + .map(|c| { + CommitInfo { + hash: c.hash(), + timestamp: c.timestamp(), + metadata: c + .message() + .and_then(|m| serde_json::from_str::(&m).ok()), + synced: false, // set later + summary: "".to_string(), // set later + } + }) + .collect::>() + }) + }) + .await + .unwrap(); + + // Check to see what the most recent synced commit is. + let mut synced_until_index = -1; + for (i, change) in changes.iter().enumerate() { + if last_acked_heads + .as_ref() + .is_some_and(|f| f.contains(&change.hash)) + { + synced_until_index = i as i32; + } + } + + let mut commit_infos = Vec::new(); + + for (i, change) in changes.into_iter().enumerate() { + // only consider changes on the current branch with valid metadata + let Some(metadata) = &change.metadata else { + continue; + }; + let Some(branch_id) = &metadata.branch_id else { + continue; + }; + if branch_id != doc_id { + continue; + } + + let summary = self + .get_change_summary(&change) + .await + .unwrap_or("Invalid data".to_string()); + let commit_info = CommitInfo { + synced: (i as i32) <= synced_until_index, + summary, + ..change + }; + commit_infos.push(commit_info); + } + commit_infos + } +} diff --git a/rust/src/project/document_watcher.rs b/rust/src/project/document_watcher.rs index baa63a17..06b28115 100644 --- a/rust/src/project/document_watcher.rs +++ b/rust/src/project/document_watcher.rs @@ -52,12 +52,13 @@ impl DocumentWatcher { let inner_clone = inner.clone(); + // do the initial ingest + inner_clone + .ingest_metadata_document(metadata_handle.clone()) + .await; + + // track changes for future ingests tokio::spawn(async move { - // do the initial ingest - inner_clone - .ingest_metadata_document(metadata_handle.clone()) - .await; - // track changes for future ingests inner_clone.track_metadata_document(metadata_handle).await; }); @@ -100,6 +101,7 @@ impl DocumentWatcherInner { } } + #[tracing::instrument(skip_all)] async fn ingest_branch_document(&self, handle: DocHandle) { let (_, meta) = self.branch_db.get_metadata_state().await.expect( "Somehow, we haven't loaded a metadata doc, but we're ingesting a branch document?!?!", @@ -207,6 +209,7 @@ impl DocumentWatcherInner { branch_state.linked_doc_ids = linked_docs.values().cloned().collect(); } + #[tracing::instrument(skip_all)] async fn ingest_metadata_document(&self, handle: DocHandle) { // TODO: Stop tracking removed branches // Find added branches, and begin tracking them diff --git a/rust/src/project/driver.rs b/rust/src/project/driver.rs index e0157f31..1a27df49 100644 --- a/rust/src/project/driver.rs +++ b/rust/src/project/driver.rs @@ -1,36 +1,60 @@ use crate::diff::differ::ProjectDiff; use crate::fs::file_utils::FileSystemEvent; -use crate::helpers::branch::BranchState; +use crate::helpers::branch::{self, BranchState}; use crate::helpers::utils::{CommitInfo, CommitMetadata}; use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkEditorAccessor}; use crate::project::branch_db::{BranchDb, HistoryRef}; +use crate::project::change_ingester::ChangeIngester; use crate::project::connection::{RemoteConnection, RemoteConnectionEvent, RemoteConnectionStatus}; use crate::project::document_watcher::DocumentWatcher; +use crate::project::main_thread_block::{self, MainThreadBlock}; use crate::project::peer_watcher::PeerWatcher; use crate::project::sync_automerge_to_fs::SyncAutomergeToFileSystem; use crate::project::sync_fs_to_automerge::SyncFileSystemToAutomerge; use automerge::ChangeHash; -use futures::StreamExt; +use futures::channel::oneshot::Cancellation; +use futures::{Stream, StreamExt}; use samod::{ConcurrencyConfig, DocHandle, DocumentId, Repo}; +use tokio::select; +use tokio::sync::{Mutex, Notify, broadcast, mpsc, oneshot, watch}; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tokio_util::sync::CancellationToken; +use tracing::instrument; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, SystemTime}; /// The main driver for the project. /// Hooks together all the various controllers. /// When this object is constructed, it is started. When the handle is dropped, it shuts down. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct Driver { inner: Arc, + repo: Repo, + token: CancellationToken, + // receivers go outside Inner, so we don't have to mutex them + file_changes_rx: mpsc::UnboundedReceiver, } #[derive(Debug)] pub struct DriverInner { + // external synchronization + main_thread_block: MainThreadBlock, + file_changes_tx: mpsc::UnboundedSender, + ref_tx: watch::Sender>, + safe_to_update_editor: AtomicBool, + token: CancellationToken, + + // subtasks repo: Repo, #[allow(unused)] connection: RemoteConnection, branch_db: BranchDb, - peer_watcher: PeerWatcher, + peer_watcher: Arc, + change_ingester: ChangeIngester, #[allow(unused)] document_watcher: DocumentWatcher, sync_automerge_to_fs: SyncAutomergeToFileSystem, @@ -40,14 +64,10 @@ pub struct DriverInner { // differ: Differ, } -impl Drop for DriverInner { +impl Drop for Driver { fn drop(&mut self) { - // don't use tokio for this, I think that's correct? - if let Ok(handle) = tokio::runtime::Handle::try_current() { - handle.block_on(self.repo.stop()); - } else { - futures::executor::block_on(self.repo.stop()); - } + self.token.cancel(); + self.repo.stop(); } } @@ -100,6 +120,7 @@ impl Driver { /// Causes tasks to run in the background. To cancel everything, drop the handle. /// If we couldn't start the driver, [None] is returned. pub async fn new( + main_thread_block: MainThreadBlock, server_url: String, project_path: PathBuf, storage_directory: PathBuf, @@ -148,43 +169,74 @@ impl Driver { // Start the connection let connection = RemoteConnection::new(repo.clone(), server_url); let branch_db = BranchDb::new(repo.clone(), project_path, ignore_globs); - - let metadata_handle = match metadata_id { + let peer_watcher = Arc::new(PeerWatcher::new(repo.clone())); + let sync_automerge_to_fs = SyncAutomergeToFileSystem::new(branch_db.clone()); + let sync_fs_to_automerge = SyncFileSystemToAutomerge::new(branch_db.clone()); + + let metadata_handle = match &metadata_id { // If we're expecting an existing ID, try and fetch it. Some(id) => { - let Some(handle) = Self::get_metadata_handle(&repo, &id, &connection).await else { + let Some(handle) = Self::get_metadata_handle(&repo, id, &connection).await else { return None; }; handle } - // If we need to make a new ID, make a doc. - None => branch_db.create_metadata_doc().await, + // If we need to make a new ID, make a doc and check in the initial state of the filesystem. + None => { + branch_db.create_metadata_doc().await + } }; // The document watcher will auto-ingest the provided metadata handle. let document_watcher = DocumentWatcher::new(repo.clone(), branch_db.clone(), metadata_handle).await; - let peer_watcher = PeerWatcher::new(repo.clone(), branch_db.clone()); - let sync_automerge_to_fs = SyncAutomergeToFileSystem::new(branch_db.clone()); - let sync_fs_to_automerge = SyncFileSystemToAutomerge::new(branch_db.clone()); + + + // If this is a new project (i.e. we earlier made a metadata doc), check in the files. + // This has to go after the document watcher ingests the metadata doc, of course. + if metadata_id.is_none() { + sync_fs_to_automerge.checkin().await; + } + + let change_ingester = ChangeIngester::new(peer_watcher.clone(), branch_db.clone()); + change_ingester.request_ingestion(); // let differ = Differ::new(branch_db.clone()); // At this point, if we loaded an existing project, we may not have checked it out yet. // We'll discover that while processing updates, and check it out then. - Some(Driver { - inner: DriverInner { - repo, + let (file_changes_tx, file_changes_rx) = mpsc::unbounded_channel(); + let (ref_tx, _) = watch::channel(None); + let token = CancellationToken::new(); + + let this = Some(Driver { + file_changes_rx, + inner: Arc::new(DriverInner { + token: token.clone(), + main_thread_block, + file_changes_tx, + ref_tx, + safe_to_update_editor: AtomicBool::new(false), + repo: repo.clone(), connection, branch_db, peer_watcher, document_watcher, + change_ingester, sync_automerge_to_fs, sync_fs_to_automerge, // differ, - } - .into(), - }) + }), + repo, + token + }); + + // Spawn off the sync task + let inner_clone = this.as_ref().unwrap().inner.clone(); + tokio::spawn(async move { + inner_clone.sync_main().await; + }); + this } pub async fn set_username(&self, username: Option) { @@ -348,95 +400,80 @@ impl Driver { Some(state.lock().await.clone()) } - /// Returns the changes from the current branch. - pub async fn get_changes(&self) -> Vec { - let checked_out = self.inner.branch_db.get_checked_out_ref_mut().await; - let checked_out = checked_out.read().await; - let Some(checked_out) = checked_out.as_ref() else { - tracing::info!("Can't get changes; nothing checked out!"); - return Vec::new(); - }; + pub async fn get_checked_out_ref(&self) -> Option { + let checked_out_ref = self.inner.branch_db.get_checked_out_ref_mut().await; + return checked_out_ref.read().await.clone(); + } - let Some(branch_state) = self - .inner - .branch_db - .get_branch_state(&checked_out.branch) - .await - else { - tracing::info!("Can't get the checked out branch state; something must be wrong"); - return Vec::new(); - }; + pub fn set_safe_to_update_editor(&self, safe: bool) { + self.inner.safe_to_update_editor.store(safe, Ordering::Relaxed); + } - // TODO: we probably don't need to lock the branch state for this whole method - let branch_state = branch_state.lock().await; - let handle = branch_state.doc_handle.clone(); - let doc_id = handle.document_id(); + // awkward + pub fn get_filesystem_changes(&mut self) -> Vec { + let mut fs_changes = Vec::new(); + while let Ok(msg) = self.file_changes_rx.try_recv() { + fs_changes.push(msg); + } + fs_changes + } - let last_acked_heads = self - .inner - .peer_watcher - .get_server_info() - .await - .as_ref() - .and_then(|info| info.docs.get(&doc_id)) - .and_then(|state| state.last_acked_heads.clone()); - - let changes = handle.with_document(move |d| { - d.get_changes(&[]) - .to_vec() - .iter() - .map(|c| { - CommitInfo { - hash: c.hash(), - timestamp: c.timestamp(), - metadata: c - .message() - .and_then(|m| serde_json::from_str::(&m).ok()), - synced: false, // set later - summary: "".to_string(), // set later - } - }) - .collect::>() - }); + // also awkward + pub fn get_changes_rx(&self) -> watch::Receiver> { + self.inner.change_ingester.get_changes_rx() + } - // Check to see what the most recent synced commit is. - let mut synced_until_index = -1; - for (i, change) in changes.iter().enumerate() { - if last_acked_heads - .as_ref() - .is_some_and(|f| f.contains(&change.hash)) - { - synced_until_index = i as i32; + pub fn get_ref_rx(&self) -> watch::Receiver> { + self.inner.ref_tx.subscribe() + } +} + +impl DriverInner { + /// Primary sync loop. + async fn sync_main(&self) { + loop { + select! { + _ = self.token.cancelled() => {break;} + // If it lags, turn this down. Alternatively, we could use a different signal to sync. + // Will cap to only once per frame due to the guard. + _ = tokio::time::sleep(Duration::from_millis(50)) => { + self.sync().await + } } } - - changes - .into_iter() - // only changes on the current branch with valid metadata - .filter(|change| { - change - .metadata - .as_ref() - .is_some_and(|m| m.branch_id.as_ref().is_some_and(|id| id == doc_id)) - }) - .enumerate() - .map(|(i, change)| CommitInfo { - synced: (i as i32) <= synced_until_index, - ..change - }) - .collect() } - pub async fn get_checked_out_ref(&self) -> Option { - let checked_out_ref = self.inner.branch_db.get_checked_out_ref_mut().await; - return checked_out_ref.read().await.clone(); + #[instrument(skip_all)] + async fn sync(&self) { + let old_checked_out_ref = self.branch_db.get_checked_out_ref_mut().await.read().await.clone(); + // Ensure we block the main thread inside of Rust while checking out a ref. + // Very important to not allow Godot to explode while we're writing files! + { + let _guard = self.main_thread_block.wait().await; + if self.safe_to_update_editor.load(Ordering::Relaxed) { + let changes = self.sync_correct_ref().await; + for change in changes { + self.file_changes_tx.send(change).unwrap(); + } + } + } + // Apply any watched FS updates to Automerge. + // It doesn't matter if we're safe to update Godot, so this can go outside of the guard. + if self.sync_fs_to_automerge.commit().await { + self.change_ingester.request_ingestion(); + } + + let new_checked_out_ref = self.branch_db.get_checked_out_ref_mut().await.read().await.clone(); + + // If we've changed branches, send the new checked out ref. + if new_checked_out_ref.as_ref().map(|r| &r.branch) != old_checked_out_ref.as_ref().map(|r| &r.branch) { + self.ref_tx.send(new_checked_out_ref).unwrap(); + } } - /// Sync the project state as best we can. - /// Make sure not to run two of these at once, for safety. - /// (It would likely be OK state-wise but weird results might happen on the UI side.) - /// Returns a vector of filesystem changes we performed. - pub async fn sync(&self, safe_to_update_godot: bool) -> Vec { + /// If our current ref is out-of-date, try and check out a new ref. + #[instrument(skip_all)] + async fn sync_correct_ref(&self) -> Vec { // TODO (Lilith): There are inefficiencies with this strategy. // Basically, every time we save a file, it'll do a bunch of extra work. // It will first commit the changes, then it will check out the changes we just committed. @@ -444,46 +481,40 @@ impl Driver { // The same happens in reverse: when we check out a ref, it will attempt to commit and not // find any actual changes. // Maybe that's OK, we need to profile to see if it's a problem. - let changes = if safe_to_update_godot { - let goal_ref = { - let current_ref_lock = self.inner.branch_db.get_checked_out_ref_mut().await; - let current_ref_guard = current_ref_lock.read().await; - - let branch = match current_ref_guard.as_ref() { - Some(r) => r.branch.clone(), - None => { - // If we didn't find a current ref, nothing is checked out. - // Let's fix that by checking out the main branch. - let Some(main_branch) = self.inner.branch_db.get_main_branch().await else { - tracing::error!( - "No metadata doc checked out, or otherwise couldn't get main branch. Skipping sync!" - ); - return Vec::new(); - }; - main_branch - } - }; - let Some(goal_ref) = self.inner.branch_db.get_latest_ref_on_branch(&branch).await - else { - tracing::error!("Couldn't get the goal ref for branch {}", branch); - return Vec::new(); - }; - goal_ref - // guard dropped here + + let goal_ref = { + let current_ref_lock = self.branch_db.get_checked_out_ref_mut().await; + let current_ref_guard = current_ref_lock.read().await; + + let branch = match current_ref_guard.as_ref() { + Some(r) => r.branch.clone(), + None => { + // If we didn't find a current ref, nothing is checked out. + // Let's fix that by checking out the main branch. + let Some(main_branch) = self.branch_db.get_main_branch().await else { + tracing::error!( + "No metadata doc checked out, or otherwise couldn't get main branch. Skipping sync!" + ); + return Vec::new(); + }; + main_branch + } }; - // TODO (Lilith): minor problem, the checked out ref could change between these lines when the guard is dropped. - // That said, uhhhh I think that's fine? We're already syncing, so other sync methods shouldn't affect it. - // If we're doing some branch edits, like a merge or revert preview or something, that could mean we - // never checkout our desired ref... but we need to rethink this for branch swapping anyways. - // For branch swapping, we either need to checkout a ref elsewhere during a merge or something and prevent sync. - // Or, we could set a request_checkout tokio::Watch here on the driver that will check out a branch at the next opportunity. - self.inner.sync_automerge_to_fs.checkout_ref(goal_ref).await - } else { - Vec::new() + let Some(goal_ref) = self.branch_db.get_latest_ref_on_branch(&branch).await + else { + tracing::error!("Couldn't get the goal ref for branch {}", branch); + return Vec::new(); + }; + goal_ref + // guard dropped here }; - - // Apply any watched FS updates to Automerge - self.inner.sync_fs_to_automerge.commit().await; - changes + // TODO (Lilith): minor problem, the checked out ref could change between these lines when the guard is dropped. + // That said, uhhhh I think that's fine? We're already syncing, so other sync methods shouldn't affect it. + // If we're doing some branch edits, like a merge or revert preview or something, that could mean we + // never checkout our desired ref... but we need to rethink this for branch swapping anyways. + // For branch swapping, we either need to checkout a ref elsewhere during a merge or something and prevent sync. + // Or, we could set a request_checkout tokio::Watch here on the driver that will check out a branch at the next opportunity. + self.sync_automerge_to_fs.checkout_ref(goal_ref).await } + } diff --git a/rust/src/project/main_thread_block.rs b/rust/src/project/main_thread_block.rs new file mode 100644 index 00000000..be46c6cc --- /dev/null +++ b/rust/src/project/main_thread_block.rs @@ -0,0 +1,79 @@ +use std::sync::{ + Arc, atomic::{AtomicU32, Ordering} +}; +use tokio::sync::{Semaphore}; + +#[cfg(test)] +mod test; + +/// A lock that blocks a main thread with checkpoint() until all worker threads have dropped their guards from wait(). +/// Worker threads also cannot begin execution until main has reached a checkpoint. +/// This has the effect of blocking the main thread until all subscribed worker threads have completed. +// TODO (Lilith): Find a concurrency expert at the lab who can help me validate/prove this implementation. I think it works, but I'm not sure. +#[derive(Clone, Debug)] +pub struct MainThreadBlock { + start: Arc, + done: Arc, + registered: Arc, +} + +impl MainThreadBlock { + pub fn new() -> Self { + Self { + // Allows us to let N threads through the gate, based on however many have registered. + start: Arc::new(Semaphore::new(0)), + // The main thread acquires N permits from done, based on how many threads it's expecting to do work. + done: Arc::new(Semaphore::new(0)), + // Tracks how many threads want through the gate this checkpoint. + registered: Arc::new(AtomicU32::new(0)), + } + } + + /// Called by worker threads. Acquires a Guard that forces a checkpoint to begin before it can run. + /// Checkpoint will block until all locked tasks are complete. + pub async fn wait(&self) -> MainThreadGuard { + // Register for this round. + // Once a round is started, we can't get into the main thread and will have to wait til next frame. + self.registered.fetch_add(1, Ordering::AcqRel); + + // Wait until main opens the gate + let permit = self.start.clone().acquire_owned().await.unwrap(); + + MainThreadGuard { + lock: self.clone(), + permit: Some(permit) + } + } + + /// Called by the main thread. If there are waiting threads asking to block the main thread with lock(), + /// releases work permits to the tasks, and does not resolve until those tasks have been completed. + /// This MUST be called by only one thread at a time!!!!!!!!! + pub async fn checkpoint(&self) { + // Snapshot how many workers registered before the cutoff + let n = self.registered.swap(0, Ordering::AcqRel); + + // Open the gate for exactly those workers + self.start.add_permits(n as usize); + + // Wait until all of them finish. We decrease the number of available permits by n each time via forgetting + // in order to ensure we hit 0 permits at the end, so we don't accidentally allow done() through prematurely + // next time. + self.done.acquire_many(n.into()).await.unwrap().forget(); + } +} + +pub struct MainThreadGuard { + lock: MainThreadBlock, + permit: Option, +} + +impl Drop for MainThreadGuard { + fn drop(&mut self) { + // We must forget the permit, to decrease the permit capacity back to 0. + // This way, when a checkpoint hits, we can still add N permits. + let permit = self.permit.take(); + permit.unwrap().forget(); + // Free the main thread by 1 permit + self.lock.done.add_permits(1); + } +} \ No newline at end of file diff --git a/rust/src/project/main_thread_block/test.rs b/rust/src/project/main_thread_block/test.rs new file mode 100644 index 00000000..73defd57 --- /dev/null +++ b/rust/src/project/main_thread_block/test.rs @@ -0,0 +1,42 @@ +use std::{sync::{Arc, atomic::{AtomicU32, Ordering}}, time::Duration}; +use tokio::time::sleep; + +use crate::project::main_thread_block::MainThreadBlock; + +#[tokio::test] +async fn test_no_workers() { + let lock = Arc::new(MainThreadBlock::new()); + // Checkpoint should complete immediately with no workers + lock.checkpoint().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_multiple_workers() { + let lock = Arc::new(MainThreadBlock::new()); + let mut workers = vec![]; + let num_completed = Arc::new(AtomicU32::new(0)); + + for iteration in 0..50 { + for i in 0..5 { + let lock_clone = Arc::clone(&lock); + let num_completed_clone = num_completed.clone(); + workers.push(tokio::spawn(async move { + let _guard = lock_clone.wait().await; + sleep(Duration::from_millis(3 * i)).await; + num_completed_clone.fetch_add(1, Ordering::Relaxed); + })); + } + + // Main thread checkpoint. Sleep first to ensure all workers are awaiting their guard. + sleep(Duration::from_millis(50)).await; + + // No workers should have run yet. + assert_eq!(num_completed.load(Ordering::Relaxed), iteration * 5); + + // Now that we're blocking in main in the correct place, run all the workers. + lock.checkpoint().await; + + // Once we're done waiting, they should've all completed. + assert_eq!(num_completed.load(Ordering::Relaxed), iteration * 5 + 5); + } +} \ No newline at end of file diff --git a/rust/src/project/peer_watcher.rs b/rust/src/project/peer_watcher.rs index f69ba292..c8b64622 100644 --- a/rust/src/project/peer_watcher.rs +++ b/rust/src/project/peer_watcher.rs @@ -1,62 +1,69 @@ -use std::sync::Arc; - -use crate::{project::branch_db::BranchDb}; -use futures::StreamExt; -use samod::{Connection, ConnectionInfo, Repo}; -use tokio::{sync::Mutex, task::JoinHandle}; +use futures::{Stream, StreamExt}; +use samod::{ConnectionInfo, Repo}; +use tokio::{select, sync::watch}; +use tokio_stream::wrappers::WatchStream; +use tokio_util::sync::CancellationToken; #[derive(Debug)] pub struct PeerWatcher { - server_info: Arc>>, - handle: JoinHandle<()>, + server_info_tx: watch::Sender>, + token: CancellationToken, } impl Drop for PeerWatcher { fn drop(&mut self) { - // Is this safe? Alternatively we could use a cancellation token - // I think it's safe though - self.handle.abort(); + self.token.cancel() } } impl PeerWatcher { - pub fn new(repo_handle: Repo, branch_db: BranchDb) -> Self { - let server_info = Arc::new(Mutex::new(None)); - let server_info_clone = server_info.clone(); + pub fn new(repo_handle: Repo) -> Self { + let (tx, rx) = watch::channel(None); + let tx_clone = tx.clone(); let repo_handle_clone = repo_handle.clone(); - let handle = tokio::spawn(async move { + let token = CancellationToken::new(); + let token_clone = token.clone(); + tokio::spawn(async move { let (_, stream) = repo_handle_clone.connected_peers(); tokio::pin!(stream); - while let Some(peers) = stream.next().await { - // Currently, we only ever have 1 peer: the server. - // Therefore, this code expects that the server is the first and only peer, if it's connected. - // When we move to more peers, we'll need to figure out a way to identify the server here. - if let Some(info) = peers.first() { - Self::update_server_info(server_info_clone.clone(), info.clone()).await; + loop { + select! { + _ = token_clone.cancelled() => { break; } + Some(peers) = stream.next() => { + // Currently, we only ever have 1 peer: the server. + // Therefore, this code expects that the server is the first and only peer, if it's connected. + // When we move to more peers, we'll need to figure out a way to identify the server here. + if let Some(info) = peers.first() { + let old_info = rx.borrow().clone(); + _ = tx_clone.send(Some(Self::update_server_info(old_info, info.clone()).await)); + } + } } } }); Self { - server_info, - handle, + server_info_tx: tx, + token, } } - pub async fn get_server_info(&self) -> Option { - return self.server_info.lock().await.clone(); + pub fn subscribe(&self) -> impl Stream> { + return WatchStream::new(self.server_info_tx.subscribe()); + } + + pub fn get_server_info(&self) -> Option { + return self.server_info_tx.subscribe().borrow().clone(); } async fn update_server_info( - old_info: Arc>>, + old_info: Option, new_info: ConnectionInfo, - ) { - let mut server_info = old_info.lock().await; - if server_info.is_none() { - *server_info = Some(new_info); - return; + ) -> ConnectionInfo { + if old_info.is_none() { + return new_info; } - let mut info = server_info.clone().unwrap(); + let mut info = old_info.unwrap(); info.last_received = new_info.last_received; info.last_sent = new_info.last_sent; @@ -77,7 +84,6 @@ impl PeerWatcher { } info.docs.insert(doc_id.clone(), new_doc_state.clone()); } - - *server_info = Some(info); + info } } diff --git a/rust/src/project/new_project.rs b/rust/src/project/project.rs similarity index 56% rename from rust/src/project/new_project.rs rename to rust/src/project/project.rs index 24d21538..e7991e6b 100644 --- a/rust/src/project/new_project.rs +++ b/rust/src/project/project.rs @@ -4,12 +4,15 @@ use crate::helpers::utils::{CommitInfo, summarize_changes}; use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkConfigAccessor, PatchworkEditorAccessor}; use crate::project::branch_db::HistoryRef; use crate::project::driver::Driver; +use crate::project::main_thread_block::MainThreadBlock; use crate::project::project_api::ChangeViewModel; use automerge::ChangeHash; use futures::future::join_all; use samod::DocumentId; +use tokio::sync::{Mutex, watch}; use std::cell::RefCell; use std::path::PathBuf; +use std::sync::Arc; use std::time::SystemTime; use std::{collections::HashMap, str::FromStr}; use tokio::runtime::Runtime; @@ -18,16 +21,22 @@ use tokio::runtime::Runtime; /// Its API is exposed to GDScript via the GodotProject struct. #[derive(Debug)] pub struct Project { - // Project driver. If some, is running - pub(super) driver: Option, + // Sync + main_thread_block: MainThreadBlock, + // These are here so we don't needlessly block during process + changes_rx: Option>>, + checked_out_ref_rx: Option>>, + + // Project driver. If some, is running. + // I'd prefer this not be a mutex, but we need to move it into temporary threads in order to dispatch async code from sync code. + // What's annoying is that we never actually block on this mutex! + pub(super) driver: Arc>>, project_dir: PathBuf, pub(super) runtime: Runtime, // Tracked changes for the UI pub(super) history: Vec, pub(super) changes: HashMap, - last_ingest: (SystemTime, i32), - ingest_requested: bool, last_known_branch: Option, // Cached diffs between refs @@ -55,47 +64,21 @@ impl Project { .unwrap(); Self { - driver: None, + main_thread_block: MainThreadBlock::new(), + changes_rx: None, + checked_out_ref_rx: None, + driver: Arc::new(Mutex::new(None)), project_dir, runtime, history: Vec::new(), changes: HashMap::new(), - last_ingest: (SystemTime::UNIX_EPOCH, 0), - ingest_requested: true, last_known_branch: None, diff_cache: RefCell::new(HashMap::new()), } } - /// Expensive operation to ingest all branch changes from automerge into the project data. - /// Should be called when we think there are new changes to process. - fn ingest_changes(&mut self) { - let Some(driver) = self.driver.clone() else { - tracing::error!("Driver not started, can't ingest changes!"); - return; - }; - - // tracing::info!("Ingesting changes..."); - - let changes = self - .runtime - .block_on(self.runtime.spawn(async move { - let changes = - driver - .get_changes() - .await - .into_iter() - .map(async |change| CommitInfo { - summary: Self::get_change_summary(&change, driver.clone()) - .await - .unwrap_or("Invalid data".to_string()), - ..change - }); - // a little awk. Parallelization doesn't really matter here at all and is maybe bad (bc locks). - // but otherwise we're iteratively constructing a vec, probably doesn't matter - join_all(changes).await - })) - .unwrap(); + fn ingest_changes(&mut self, changes: Vec) { + tracing::info!("Ingesting changes..."); self.history.clear(); self.changes.clear(); @@ -106,72 +89,7 @@ impl Project { self.changes.insert(change.hash, change); } } - - async fn get_change_summary(change: &CommitInfo, driver: Driver) -> Option { - let meta = change.metadata.as_ref(); - let author = meta?.username.clone().unwrap_or("Anonymous".to_string()); - - // merge commit - if let Some(merge_info) = &meta?.merge_metadata { - let merged_branch = driver - .get_branch_name(&merge_info.merged_branch_id.clone()) - .await - .unwrap_or(merge_info.merged_branch_id.to_string()); - return Some(format!("↪ {author} merged {merged_branch}")); - } - - // revert commit - if let Some(revert_info) = &meta?.reverted_to { - let heads = revert_info - .iter() - .map(|s| &s[..7]) - .collect::>() - .join(", "); - return Some(format!("↩ {author} reverted to {heads}")); - } - - // initial commit - if change.is_setup() { - return Some(format!("Initialized repository")); - } - - return Some(summarize_changes(&author, meta?.changed_files.as_ref()?)); - } - - /// Request for a change ingestion to be dispatched. - fn request_ingestion(&mut self) { - self.ingest_requested = true; - } - - /// If able, ingest changes, clear the ingestion request, and return true. - /// Otherwise, return false. - fn try_ingest_changes(&mut self) -> bool { - // Do not try to ingest if we haven't requested. - if !self.ingest_requested { - return false; - } - let now = SystemTime::now(); - let Ok(last_diff) = now.duration_since(self.last_ingest.0) else { - return false; - }; - - // Impose an arbitrary cap on requests within a time period. - // This is so that immediate syncs -- such as those from a local server -- don't have to wait before getting synced. - // But it also prevents spam of like a hundred slowing down the ingestion. - if last_diff.as_millis() < 100 { - if self.last_ingest.1 >= 3 { - return false; - } - } else { - // since we're past the duration with no other requests, the counter resets. - self.last_ingest = (now, 0); - } - self.ingest_changes(); - self.ingest_requested = false; - self.last_ingest.1 += 1; - true - } - + pub fn get_cached_diff(&self, before: HistoryRef, after: HistoryRef) -> ProjectDiff { self.diff_cache .borrow_mut() @@ -193,7 +111,7 @@ impl Project { } pub fn get_diff(&self, before: HistoryRef, after: HistoryRef) -> ProjectDiff { - let Some(driver) = &self.driver else { + let Some(driver) = self.driver.blocking_lock().as_ref() else { return ProjectDiff::default(); }; ProjectDiff::default() @@ -208,7 +126,7 @@ impl Project { } pub fn start(&mut self) { - if self.driver.is_some() { + if self.driver.blocking_lock().is_some() { tracing::error!("Driver is already started!"); return; } @@ -256,57 +174,78 @@ impl Project { ); let project_dir = self.project_dir.clone(); - self.driver = self + let block = self.main_thread_block.clone(); + + // TODO: Don't block on main thread for checkin + *self.driver.blocking_lock() = self .runtime .block_on( // I think it's correct to spawn this on a different task explicitly, because block_on runs the future on the current thread, not a worker thread. self.runtime.spawn(async move { - Driver::new(server_url, project_dir, storage_dir, metadata_id).await + Driver::new(block, server_url, project_dir, storage_dir, metadata_id).await }), ) .unwrap(); - if self.driver.is_none() { + if self.driver.blocking_lock().is_none() { tracing::error!("Could not start the driver!"); return; } + + let driver = self.driver.blocking_lock(); + self.changes_rx = Some(driver.as_ref().unwrap().get_changes_rx()); + self.checked_out_ref_rx = Some(driver.as_ref().unwrap().get_ref_rx()); } pub fn stop(&mut self) { - self.driver.take(); + self.driver.blocking_lock().take(); } pub fn process(&mut self, _delta: f64) -> (Vec, Vec) { - let Some(driver) = &self.driver else { - return Default::default(); + let fs_changes = { + let mut driver_guard = self.driver.blocking_lock(); + if driver_guard.is_none() { + return (Vec::new(), Vec::new()); + } + // Run the blocking sync + driver_guard.as_ref().unwrap().set_safe_to_update_editor(Self::safe_to_update_godot()); + let block = self.main_thread_block.clone(); + self.runtime.block_on(self.runtime.spawn(async move { + block.checkpoint().await; + })).unwrap(); + + // Consume any modified files to send to Godot + driver_guard.as_mut().unwrap().get_filesystem_changes() }; - let driver = driver.clone(); - - let mut signals: Vec = Vec::new(); - if self.try_ingest_changes() { - signals.push(GodotProjectSignal::ChangesIngested); + + let mut signals = Vec::new(); + + // Ingest changes if the driver produced a new changeset + let changes = { + let rx = self.changes_rx.as_mut().unwrap(); + if rx.has_changed().unwrap_or(false) { + rx.mark_unchanged(); + signals.push(GodotProjectSignal::ChangesIngested); + Some(rx.borrow().clone()) + } + else { + None + } + }; + + if let Some(changes) = changes { + self.ingest_changes(changes); } - let safe_to_update = Self::safe_to_update_godot(); - // Run the main sync - let (changed_files, checked_out_ref) = - self.runtime - .block_on(self.runtime.spawn(async move { - (driver.sync(safe_to_update).await, driver.get_checked_out_ref().await) - })) - .unwrap(); - let current_branch = checked_out_ref.and_then(|r| Some(r.branch)); - if self.last_known_branch != current_branch { + // Check to see if we need to produce a CheckedOutBranch signal + let rx = self.checked_out_ref_rx.as_ref().unwrap(); + if rx.has_changed().unwrap_or(false) { signals.push(GodotProjectSignal::CheckedOutBranch); - self.last_known_branch = current_branch; } // TODO (Lilith): VERY IMPORTANT, set the patchwork config branch ID here!!! // So that we save the branch ID for future checkouts. - // TODO (Lilith): Don't request an ingestion every frame - self.request_ingestion(); - - (changed_files, signals) + (fs_changes, signals) } } diff --git a/rust/src/project/project_api_impl.rs b/rust/src/project/project_api_impl.rs index addf5c19..6f3f3419 100644 --- a/rust/src/project/project_api_impl.rs +++ b/rust/src/project/project_api_impl.rs @@ -13,7 +13,7 @@ use crate::{ interop::godot_accessors::PatchworkConfigAccessor, project::{ branch_db::HistoryRef, - new_project::Project, + project::Project, project_api::{ BranchViewModel, ChangeViewModel, DiffViewModel, ProjectViewModel, SyncStatus, }, @@ -23,15 +23,22 @@ use crate::{ // TODO: Ideally this is actually a child of a new project submodule... // that's so that it doesn't need pub(super) to acess private fields of // itself. +// TODO (Lilith): Figure out if there's a reasonable way to reduce blocking in this file. impl ProjectViewModel for Project { fn has_project(&self) -> bool { - self.driver.is_some() + self.driver.blocking_lock().is_some() } fn get_project_id(&self) -> Option { - let driver = self.driver.clone()?; + let driver = self.driver.clone(); self.runtime - .block_on(self.runtime.spawn(async move { driver.get_metadata_doc().await })) + .block_on(self.runtime.spawn(async move { + let driver = driver.lock().await; + if driver.is_none() { + return None; + } + driver.as_ref().unwrap().get_metadata_doc().await + })) .unwrap() } @@ -70,12 +77,16 @@ impl ProjectViewModel for Project { fn set_user_name(&self, name: String) { PatchworkConfigAccessor::set_user_value("user_name", &name); - let Some(driver) = self.driver.clone() else { - return; - }; - self.runtime.block_on(self.runtime.spawn(async move { - driver.set_username(Some(name)).await - })).unwrap(); + let driver = self.driver.clone(); + self.runtime + .block_on(self.runtime.spawn(async move { + let mut driver = driver.lock().await; + if driver.is_none() { + return; + } + driver.as_mut().unwrap().set_username(Some(name)).await + })) + .unwrap(); } fn can_create_merge_preview_branch(&self) -> bool { @@ -291,13 +302,17 @@ impl ProjectViewModel for Project { } fn get_branch(&self, id: &DocumentId) -> Option> { - let driver = self.driver.clone()?; + let driver = self.driver.clone(); let id = id.clone(); let state = self .runtime - .block_on(self.runtime.spawn( - async move { driver.get_branch_state(&id).await }, - )) + .block_on(self.runtime.spawn(async move { + let driver = driver.lock().await; + if driver.is_none() { + return None; + } + driver.as_ref().unwrap().get_branch_state(&id).await + })) .unwrap(); let Some(state) = state else { return None; @@ -338,21 +353,31 @@ impl ProjectViewModel for Project { } fn get_main_branch(&self) -> Option { - let driver = self.driver.clone()?; + let driver = self.driver.clone(); let id = self .runtime - .block_on(self.runtime.spawn(async move { driver.get_main_branch().await })) + .block_on(self.runtime.spawn(async move { + let driver = driver.lock().await; + if driver.is_none() { + return None; + } + driver.as_ref().unwrap().get_main_branch().await + })) .unwrap()?; self.get_branch(&id) } fn get_checked_out_branch(&self) -> Option { - let driver = self.driver.clone()?; + let driver = self.driver.clone(); let id = self .runtime - .block_on(self.runtime.spawn( - async move { driver.get_checked_out_ref().await }, - )) + .block_on(self.runtime.spawn(async move { + let driver = driver.lock().await; + if driver.is_none() { + return None; + } + driver.as_ref().unwrap().get_checked_out_ref().await + })) .unwrap()?; self.get_branch(&id.branch) } @@ -434,12 +459,18 @@ impl ProjectViewModel for Project { fn get_default_diff(&self) -> Option { let heads_before; let heads_after; - let driver = self.driver.clone()?; + let driver = self.driver.clone(); let branch_state = self .runtime .block_on(self.runtime.spawn(async move { + let driver = driver.lock().await; + if driver.is_none() { + return None; + } driver - .get_branch_state(&driver.get_checked_out_ref().await?.branch) + .as_ref() + .unwrap() + .get_branch_state(&driver.as_ref().unwrap().get_checked_out_ref().await?.branch) .await })) .unwrap()?; @@ -528,12 +559,18 @@ impl ProjectViewModel for Project { } } - let driver = self.driver.clone()?; + let driver = self.driver.clone(); let branch_state = self .runtime .block_on(self.runtime.spawn(async move { + let driver = driver.lock().await; + if driver.is_none() { + return None; + } driver - .get_branch_state(&driver.get_checked_out_ref().await?.branch) + .as_ref() + .unwrap() + .get_branch_state(&driver.as_ref().unwrap().get_checked_out_ref().await?.branch) .await })) .unwrap()?; diff --git a/rust/src/project/sync_automerge_to_fs.rs b/rust/src/project/sync_automerge_to_fs.rs index 30d23cbc..1264b73a 100644 --- a/rust/src/project/sync_automerge_to_fs.rs +++ b/rust/src/project/sync_automerge_to_fs.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; use futures::future::join_all; +use tracing::instrument; use crate::{ fs::{file_utils::FileContent, file_utils::FileSystemEvent}, @@ -25,6 +26,7 @@ impl SyncAutomergeToFileSystem { /// Check out a [HistoryRef] from the Patchwork history, changing the filesystem as necessary. /// Returns a vector of file changes. + #[instrument(skip_all)] pub async fn checkout_ref(&self, goal_ref: HistoryRef) -> Vec { // Ensure that there's no way anything can grab the ref while we're trying to write it let r = self.branch_db.get_checked_out_ref_mut().await; diff --git a/rust/src/project/sync_fs_to_automerge.rs b/rust/src/project/sync_fs_to_automerge.rs index c6a7c6d7..038fe435 100644 --- a/rust/src/project/sync_fs_to_automerge.rs +++ b/rust/src/project/sync_fs_to_automerge.rs @@ -1,10 +1,9 @@ use std::{path::PathBuf, sync::Arc}; use futures::StreamExt; -use tokio::{ - select, sync::Mutex, task::{JoinHandle, JoinSet} -}; +use tokio::{select, sync::Mutex, task::JoinSet}; use tokio_util::sync::CancellationToken; +use tracing::instrument; use crate::{ fs::{file_utils::FileContent, file_utils::FileSystemEvent}, @@ -76,7 +75,9 @@ impl SyncFileSystemToAutomerge { } /// Make a commit of all watched, pending changes from the filesystem to automerge. - pub async fn commit(&self) { + /// Returns true on success. + #[instrument(skip_all)] + pub async fn commit(&self) -> bool { // Because we always change the checked out ref after committing, we need to lock this in write mode. let r = self.branch_db.get_checked_out_ref_mut().await; let mut checked_out_ref = r.write().await; @@ -84,15 +85,21 @@ impl SyncFileSystemToAutomerge { let mut pending_changes = self.pending_changes.lock().await; if pending_changes.is_empty() { - return; + return false; } - tracing::info!("There are {:?} pending changes, attempting to commit...", pending_changes.len()); + tracing::info!( + "There are {:?} pending changes, attempting to commit...", + pending_changes.len() + ); // If the checked-out ref is invalid, we can't commit to the current branch. if checked_out_ref.as_ref().is_none_or(|r| !r.is_valid()) { - tracing::warn!("Can't commit to the current ref {:?}, because it isn't valid.", checked_out_ref); - return; + tracing::warn!( + "Can't commit to the current ref {:?}, because it isn't valid.", + checked_out_ref + ); + return false; } let new_ref = self @@ -108,18 +115,27 @@ impl SyncFileSystemToAutomerge { tracing::info!("Successfully made a commit! {:?}", new_ref); pending_changes.clear(); *checked_out_ref = Some(new_ref); + return true; } else { tracing::error!("Could not commit pending files! Making no changes."); + return false; } } - // TODO (Lilith): We need to check in the files... make it happen. /// Make an initial commit of ALL files from the filesystem to automerge. + /// Makes the commit on the currently checked-out branch, and checks out the new heads. pub async fn checkin(&self) { // Because we always change the checked out ref after committing, we need to lock this in write mode. let r = self.branch_db.get_checked_out_ref_mut().await; let mut checked_out_ref = r.write().await; + if checked_out_ref.is_none() { + tracing::error!("Could not check in files; we don't have a branch checked out!"); + } + else { + tracing::info!("Checking in files at ref {:?}", checked_out_ref); + } + let files = self.get_all_files().await; let new_ref = self @@ -137,12 +153,6 @@ impl SyncFileSystemToAutomerge { } else { tracing::error!("Could not check in files! Making no changes."); } - // The original code runs a sync here... I'm not sure why. The equivalent in the new system would be SyncAutomergeToFileSystem. - // I think because we're updating the heads, we're OK... - // self.sync_files_at( - // branch_state.doc_handle.clone(), - // files.into_iter().map(|(path, content)| (PathBuf::from(path), content)).collect::>(), - // Some(branch_state.synced_heads.clone()), } fn get_all_files_recur( diff --git a/rust/tests/test.rs b/rust/tests/test.rs deleted file mode 100644 index dcfea7b3..00000000 --- a/rust/tests/test.rs +++ /dev/null @@ -1,394 +0,0 @@ -use std::collections::HashMap; -use std::collections::HashSet; - -use automerge::Automerge; -// Import the modules from the library -use patchwork_rust_core::godot_parser; -use patchwork_rust_core::godot_parser::GodotConnection; -use patchwork_rust_core::godot_parser::GodotScene; -// use pretty_assertions::{assert_eq, assert_ne}; - -fn get_test_scene_source() -> String { - r#"[gd_scene load_steps=6 format=3 uid="uid://jnrusvm3gric"] - -[ext_resource type="Texture2D" uid="uid://dw612tw7iymyb" path="res://assets/background-layer-1.png" id="1_0qn5k"] -[ext_resource type="Script" path="res://scripts/background.gd" id="1_1jh5j"] -[ext_resource type="Texture2D" uid="uid://dne1wh5fsffy" path="res://assets/background-layer-2.png" id="2_mk66l"] - -[sub_resource type="Gradient" id="Gradient_80myt"] -colors = PackedColorArray(0.98, 0.98, 0.98, 1, 0.81, 0.81, 0.81, 1) -offsets = PackedFloat32Array(0.0788732, 1) - -[sub_resource type="GradientTexture2D" id="GradientTexture2D_ljotv"] -fill_to = Vector2(0, 1) -gradient = SubResource("Gradient_80myt") -height = 1080 -width = 5115 - -[node name="Background" type="ParallaxBackground"] -follow_viewport_enabled = true -script = ExtResource("1_1jh5j") -scroll_ignore_camera_zoom = true -metadata/patchwork_id = "1122ae43c1054005997967892c521ea9" - -[node name="ParallaxLayer" type="ParallaxLayer" parent="."] -motion_mirroring = Vector2(1600, 0) -motion_scale = Vector2(0, 0) -unique_name_in_owner = true -metadata/patchwork_id = "ae876d398eb24a959b9ff1b00d983948" - -[node name="ColorRect" type="ColorRect" parent="ParallaxLayer"] -offset_bottom = 98.0 -offset_right = 5129.0 -offset_top = -2352.0 -anchor_bottom = 1.0 -anchor_right = 1.0 -anchors_preset = 15 -color = Color(0.980392, 0.980392, 0.980392, 1) -grow_horizontal = 2 -grow_vertical = 2 -metadata/patchwork_id = "5b9416e8d96042b6a509f7da3263f687" - -[node name="NestedColorRect" type="ColorRect" parent="ParallaxLayer/ColorRect"] -anchor_bottom = 1.0 -anchor_right = 1.0 -anchors_preset = 15 -color = Color(0.980392, 0.980392, 0.980392, 1) -grow_horizontal = 2 -grow_vertical = 2 -offset_bottom = 98.0 -offset_right = 5129.0 -offset_top = -2352.0 -metadata/patchwork_id = "9a7c3e5b8f2d1a6c4b8e9d7f5a3c1e8b" - -[node name="Sprite2D" type="Sprite2D" parent="ParallaxLayer"] -centered = false -texture = SubResource("GradientTexture2D_ljotv") -metadata/patchwork_id = "50a6b8d7ce2c469098b3416372f9b1b8" - -"#.to_string() -} - -#[test] -fn test_parse_and_serialize() { - let source = get_test_scene_source(); - - // Parse the scene - let scene = godot_parser::parse_scene(&source).unwrap(); - - // Serialize the scene back to string - let reserialized = scene.serialize(); - - // Verify that the serialized output matches the original input - assert_eq!( - source, reserialized, - "Serialized output should match original input" - ); -} - -fn get_test_scene_source_with_connections_source() -> String { - r#"[gd_scene load_steps=6 format=3 uid="uid://jnrusvm3gric"] - -[node name="Root" type="Node2D"] - -[node name="GameManager" type="Node2D" parent="."] - -[node name="UI" type="Node2D" parent="."] - -[node name="Button" type="Button" parent="UI"] - -[connection signal="button_pressed" from="UI/Button" to="GameManager" method="_on_button_pressed" flags=3 unbinds=1 binds=["extra_param", 42, true]]"#.to_string() -} - -#[test] -fn test_parse_scene_with_connections() { - let source = get_test_scene_source_with_connections_source(); - let scene = godot_parser::parse_scene(&source).unwrap(); - - let game_manager_node = scene - .nodes - .values() - .find(|node| node.name == "GameManager") - .unwrap(); - let button_node = scene - .nodes - .values() - .find(|node| node.name == "Button") - .unwrap(); - - let connections = scene.connections.values().collect::>(); - let connection = connections[0]; - - assert_eq!(connections.len(), 1); - assert_eq!(connection.signal, "button_pressed"); - assert_eq!(connection.from_node_id, button_node.id); - assert_eq!(connection.to_node_id, game_manager_node.id); - assert_eq!(connection.method, "_on_button_pressed"); - assert_eq!(connection.flags, Some(3)); - assert_eq!(connection.unbinds, Some(1)); - assert_eq!( - connection.binds, - Some("[\"extra_param\", 42, true]".to_string()) - ); -} - -#[test] -fn test_parse_scene_with_duplicate_ids() { - let source = r#"[gd_scene load_steps=6 format=3 uid="uid://jnrusvm3gric"] -[node name="Root" type="Node2D"] -metadata/patchwork_id = "1122ae43c1054005997967892c521ea9" - -[node name="ColorRect1" type="ColorRect" parent="."] -color = Color(0.980392, 0.980392, 0.980392, 1) -metadata/patchwork_id = "5b9416e8d96042b6a509f7da3263f687" - -[node name="ColorRect2" type="ColorRect" parent="."] -color = Color(0.980392, 0.980392, 0.980392, 1) -metadata/patchwork_id = "5b9416e8d96042b6a509f7da3263f687" -"# - .to_string(); - - let scene = godot_parser::parse_scene(&source).unwrap(); - - let root_node = scene.nodes.get(scene.root_node_id.as_ref().unwrap()).unwrap(); - - assert_eq!(root_node.child_node_ids.len(), 2); - - println!("root_node: {:?}", root_node.child_node_ids); - - // first node has original id - assert_eq!( - root_node.child_node_ids[0], - "5b9416e8d96042b6a509f7da3263f687" - ); - - // second node gets assigned a new id - assert_ne!( - root_node.child_node_ids[1], - "5b9416e8d96042b6a509f7da3263f687" - ); - - assert_eq!( - scene - .nodes - .get(root_node.child_node_ids[1].as_str()) - .unwrap() - .id, - root_node.child_node_ids[1], - ); -} - -#[test] -fn test_resconcile_and_hydrate() { - let example_scene = godot_parser::GodotScene { - format: 3, - load_steps: 0, - uid: "uid://b8vp42c3k4q7v".to_string(), - script_class: None, - resource_type: "packed_scene".to_string(), - editable_instances: HashSet::new(), - main_resource: None, - nodes: HashMap::from([ - ( - "node1".to_string(), - godot_parser::GodotNode { - name: "Root".to_string(), - parent_id: None, - properties: HashMap::new(), - id: "node1".to_string(), - type_or_instance: godot_parser::TypeOrInstance::Type("Node2D".to_string()), - owner: None, - index: None, - groups: None, - child_node_ids: vec!["node2".to_string(), "node3".to_string()], - node_paths: None, - }, - ), - ( - "node2".to_string(), - godot_parser::GodotNode { - name: "Sprite".to_string(), - parent_id: Some("node1".to_string()), - properties: HashMap::from([( - "position".to_string(), - godot_parser::OrderedProperty::new("Vector2(100.0, 100.0)".to_string(), 0), - )]), - id: "node2".to_string(), - type_or_instance: godot_parser::TypeOrInstance::Type("Sprite2D".to_string()), - owner: None, - index: None, - groups: None, - child_node_ids: vec![], - node_paths: None, - }, - ), - ( - "node3".to_string(), - godot_parser::GodotNode { - name: "Label".to_string(), - parent_id: Some("node1".to_string()), - properties: HashMap::from([ - ( - "offset_right".to_string(), - godot_parser::OrderedProperty::new("40.0".to_string(), 0), - ), - ( - "offset_bottom".to_string(), - godot_parser::OrderedProperty::new("23.0".to_string(), 1), - ), - ( - "text".to_string(), - godot_parser::OrderedProperty::new("\"Hello World\"".to_string(), 2), - ), - ]), - id: "node3".to_string(), - type_or_instance: godot_parser::TypeOrInstance::Type("Label".to_string()), - owner: None, - index: None, - groups: None, - child_node_ids: vec![], - node_paths: None, - }, - ), - ]), - root_node_id: Some("node1".to_string()), - ext_resources: HashMap::from([ - ( - "1_0qn5k".to_string(), - godot_parser::ExternalResourceNode { - idx: 0, - id: "1_0qn5k".to_string(), - path: "res://assets/background-layer-1.png".to_string(), - resource_type: "Texture2D".to_string(), - uid: Some("uid://dw612tw7iymyb".to_string()), - }, - ), - ( - "1_1jh5j".to_string(), - godot_parser::ExternalResourceNode { - idx: 1, - id: "1_1jh5j".to_string(), - path: "res://scripts/background.gd".to_string(), - resource_type: "Script".to_string(), - uid: None, - }, - ), - ]), - sub_resources: HashMap::from([( - "Gradient_80myt".to_string(), - godot_parser::SubResourceNode { - idx: 0, - id: "Gradient_80myt".to_string(), - resource_type: "Gradient".to_string(), - properties: HashMap::from([ - ( - "colors".to_string(), - godot_parser::OrderedProperty::new( - "PackedColorArray(0.98, 0.98, 0.98, 1, 0.81, 0.81, 0.81, 1)" - .to_string(), - 0, - ), - ), - ( - "offsets".to_string(), - godot_parser::OrderedProperty::new( - "PackedFloat32Array(0.0788732, 1)".to_string(), - 1, - ), - ), - ]), - }, - )]), - connections: HashMap::from([( - "my_signal-node1-node2-my_method--".to_string(), - GodotConnection { - signal: "my_signal".to_string(), - from_node_id: "node1".to_string(), - to_node_id: "node2".to_string(), - method: "my_method".to_string(), - flags: None, - unbinds: None, - binds: None, - }, - )]), - requires_resave: false, - }; - - // write to automerge doc - - let mut doc = Automerge::new(); - let mut tx = doc.transaction(); - example_scene.reconcile(&mut tx, "example.tscn".to_string()); - tx.commit(); - - // rehydrate from automerge doc - - let rehydrated_scene = GodotScene::hydrate(&mut doc, "example.tscn").unwrap(); - - assert_eq!(example_scene, rehydrated_scene) -} - - -#[test] -fn test_node_paths() { - let source = r#"[gd_scene load_steps=6 format=3 uid="uid://jnrusvm3gric"] -[node name="Root" type="Node2D"] -metadata/patchwork_id = "1122ae43c1054005997967892c521ea9" - -[node name="Child" type="Node2D" parent="."] -metadata/patchwork_id = "1122ae43c1054005997967892c521ea2" - -[node name="FindInFiles" parent="Child" node_paths=PackedStringArray("main_view", "code_edit") type="Node2D"] -metadata/patchwork_id = "1122ae43c1054005997967892c521ea3" -"#; - - let scene = godot_parser::parse_scene(&source.to_string()).unwrap(); - - assert_eq!(scene.nodes.len(), 3); - assert_eq!(scene.nodes["1122ae43c1054005997967892c521ea9"].node_paths, None); - assert_eq!(scene.nodes["1122ae43c1054005997967892c521ea2"].node_paths, None); - assert_eq!(scene.nodes["1122ae43c1054005997967892c521ea3"].node_paths, Some("PackedStringArray(\"main_view\", \"code_edit\")".to_string())); - -} - -#[test] -fn test_stringnames() { - let source = r#"[gd_resource type="AudioBusLayout" format=3 uid="uid://c7thbop54thnf"] - -[resource] -bus/1/name = &"Music" -bus/1/solo = false -bus/1/mute = false -bus/1/bypass_fx = false -bus/1/volume_db = 0.0 -bus/1/send = &"Master" -bus/2/name = &"SFX" -bus/2/solo = false -bus/2/mute = false -bus/2/bypass_fx = false -bus/2/volume_db = 0.0 -bus/2/send = &"Master" -prop:with_colon/2/name = &"Music" - -"#; - - let resource = godot_parser::parse_scene(&source.to_string()).unwrap(); - - assert_eq!(resource.main_resource.is_some(), true); - let main_resource = resource.main_resource.as_ref().unwrap(); - let prop = main_resource.properties["bus/1/name"].clone(); - let ref_prop = godot_parser::OrderedProperty::new("&\"Music\"".to_string(), 0); - assert_eq!(prop, ref_prop); - - let prop_with_colon = main_resource.properties["prop:with_colon/2/name"].clone(); - let ref_prop_with_colon = godot_parser::OrderedProperty::new("&\"Music\"".to_string(), 12); - assert_eq!(prop_with_colon, ref_prop_with_colon); - - // reserialize and string compare - let new_resource = resource.serialize(); - assert_eq!(new_resource, source); - - -} - - From 0a968383ab22d9e9a901d925bd9639b900c1bb49 Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:51:17 -0800 Subject: [PATCH 05/49] s --- public/gdscript/sidebar.gd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/gdscript/sidebar.gd b/public/gdscript/sidebar.gd index 5e9923e5..6a954421 100644 --- a/public/gdscript/sidebar.gd +++ b/public/gdscript/sidebar.gd @@ -75,7 +75,7 @@ signal reload_ui(); signal user_name_dialog_closed(); func _update_ui_on_state_change(): - # print("Patchwork: Updating UI due to state change...") + print("Patchwork: Updating UI due to state change...") update_ui() func _update_ui_on_branch_checked_out(): From fcfd8d874fc9cfe1794a5d96b3a6e83eef2f82a3 Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:56:42 -0800 Subject: [PATCH 06/49] re-enable change ingestion --- rust/src/project/change_ingester.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/src/project/change_ingester.rs b/rust/src/project/change_ingester.rs index 49290c05..9572433a 100644 --- a/rust/src/project/change_ingester.rs +++ b/rust/src/project/change_ingester.rs @@ -62,9 +62,9 @@ impl ChangeIngester { _ = stream.next() => { inner_clone.ingestion_request.notify_one(); } - // _ = inner_clone.ingestion_request.notified() => { - // inner_clone.ingest_changes().await; - // }, + _ = inner_clone.ingestion_request.notified() => { + inner_clone.ingest_changes().await; + }, } } }); From 8e93c77a3071e87b9af2b764bd2dd835c4577055 Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Mon, 26 Jan 2026 11:02:37 -0800 Subject: [PATCH 07/49] fix --- rust/src/project/change_ingester.rs | 9 +++++---- rust/src/project/driver.rs | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/rust/src/project/change_ingester.rs b/rust/src/project/change_ingester.rs index 9572433a..563e2e47 100644 --- a/rust/src/project/change_ingester.rs +++ b/rust/src/project/change_ingester.rs @@ -174,15 +174,16 @@ impl ChangeIngesterInner { let h = handle.clone(); let changes = tokio::task::spawn_blocking(move || { h.with_document(move |d| { - d.get_changes(&[]) + d.get_changes_meta(&[]) .to_vec() .iter() .map(|c| { CommitInfo { - hash: c.hash(), - timestamp: c.timestamp(), + hash: c.hash, + timestamp: c.timestamp, metadata: c - .message() + .message + .as_ref() .and_then(|m| serde_json::from_str::(&m).ok()), synced: false, // set later summary: "".to_string(), // set later diff --git a/rust/src/project/driver.rs b/rust/src/project/driver.rs index 1a27df49..090f15f8 100644 --- a/rust/src/project/driver.rs +++ b/rust/src/project/driver.rs @@ -436,7 +436,7 @@ impl DriverInner { _ = self.token.cancelled() => {break;} // If it lags, turn this down. Alternatively, we could use a different signal to sync. // Will cap to only once per frame due to the guard. - _ = tokio::time::sleep(Duration::from_millis(50)) => { + _ = tokio::time::sleep(Duration::from_millis(5)) => { self.sync().await } } From fd6e894295dbe197c835bc48c052eacc09add947 Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Mon, 26 Jan 2026 12:02:18 -0800 Subject: [PATCH 08/49] reduce stupid design in branchdb --- rust/src/project/branch_db.rs | 68 +++++++++++----------------- rust/src/project/branch_db/branch.rs | 8 ++-- rust/src/project/branch_db/commit.rs | 19 +++----- rust/src/project/branch_db/file.rs | 2 +- 4 files changed, 38 insertions(+), 59 deletions(-) diff --git a/rust/src/project/branch_db.rs b/rust/src/project/branch_db.rs index fc01dc18..804ec0d8 100644 --- a/rust/src/project/branch_db.rs +++ b/rust/src/project/branch_db.rs @@ -19,6 +19,7 @@ mod branch; mod commit; mod file; mod util; +mod merge_revert; // TODO (Lilith): Move this to utils /// Represents a location anywhere in Patchwork's history. @@ -57,27 +58,15 @@ impl PartialEq for HistoryRef { /// It stores the project state, and provides a handful of convenient state-manipulation methods for controllers to use. #[derive(Clone, Debug)] pub struct BranchDb { - // TODO (Lilith): consider swapping to RwLock - inner: Arc>, - // These are immutable, so they can be outside the mutex + // Path is immutable, so it can be outside the inner project_dir: PathBuf, ignore_globs: Arc>, -} - -#[derive(Debug)] -pub struct BranchDbInner { - // TODO (Lilith): We need to figure out a way to populate binary docs. - // The old system is hugely flawed and would be best to avoid. - // Instead, consider a helper shim class to track each binary doc as they come in, and update the fs as needed. - // But what's the intended user behavior? Consult with Nikita and Paul. - - // Does this need to be in inner? Probably not, right? repo: Repo, - username: Option, - - binary_states: HashMap, - branch_states: HashMap>>, - metadata_state: Option<(DocumentId, BranchesMetadataDoc)>, + + username: Arc>>, + binary_states: Arc>>, + branch_states: Arc>>>>, // might be too much locking + metadata_state: Arc>>, // The checked out ref is the ref that the filesystem is currently synced with. // Has a separate lock because of its importance; it needs to be locked while we're prepping a commit or checking out stuff @@ -88,15 +77,13 @@ impl BranchDb { pub fn new(repo: Repo, project_dir: PathBuf, ignore_globs: Vec) -> Self { Self { project_dir, + repo, ignore_globs: Arc::new(ignore_globs), - inner: Arc::new(Mutex::new(BranchDbInner { - repo, - username: None, - binary_states: HashMap::new(), - branch_states: HashMap::new(), - metadata_state: None, - checked_out_ref: Arc::new(RwLock::new(None)), - })), + username: Arc::new(Mutex::new(None)), + binary_states: Arc::new(Mutex::new(HashMap::new())), + branch_states: Arc::new(Mutex::new(HashMap::new())), + metadata_state: Arc::new(Mutex::new(None)), + checked_out_ref: Arc::new(RwLock::new(None)), } } @@ -110,39 +97,37 @@ impl BranchDb { } pub async fn set_username(&self, username: Option) { - let mut inner = self.inner.lock().await; - inner.username = username; + let mut user = self.username.lock().await; + *user = username; } /// Get the mutable checked out ref for locking. pub async fn get_checked_out_ref_mut(&self) -> Arc>> { - return self.inner.lock().await.checked_out_ref.clone(); + return self.checked_out_ref.clone(); } pub async fn get_metadata_state(&self) -> Option<(DocumentId, BranchesMetadataDoc)> { - let inner = self.inner.lock().await; // This is a needlessly expensive operation; we should consider allowing reference introspection via external lockers. // And/or improve clone perf by reducing string usage in BranchesMetadataDoc. - inner.metadata_state.clone() + self.metadata_state.lock().await.clone() } pub async fn set_metadata_state(&self, id: DocumentId, state: BranchesMetadataDoc) { - let mut inner = self.inner.lock().await; - inner.metadata_state = Some((id, state)); + let mut st = self.metadata_state.lock().await; + *st = Some((id, state)); } pub async fn has_branch(&self, id: &DocumentId) -> bool { - let inner = self.inner.lock().await; - return inner.branch_states.contains_key(id); + let st = self.branch_states.lock().await; + return st.contains_key(id); } pub async fn insert_branch_state_if_not_exists(&self, id: DocumentId, f: F) where F: FnOnce() -> BranchState, { - let mut inner = self.inner.lock().await; - inner - .branch_states + let mut st = self.branch_states.lock().await; + st .entry(id.clone()) .or_insert_with(|| Arc::new(Mutex::new(f()))); } @@ -151,13 +136,12 @@ impl BranchDb { // Alternatively we could provide a read-only view in a closure, or clone them. // We do need to be a little careful about locks though. pub async fn get_branch_state(&self, id: &DocumentId) -> Option>> { - let inner = self.inner.lock().await; - inner.branch_states.get(id).cloned() + let st = self.branch_states.lock().await; + st.get(id).cloned() } pub async fn get_branch_handle(&self, id: &DocumentId) -> Option { - let inner = self.inner.lock().await; - let Some(state) = inner.branch_states.get(id) else { + let Some(state) = self.get_branch_state(id).await else { return None; }; Some(state.lock().await.doc_handle.clone()) diff --git a/rust/src/project/branch_db/branch.rs b/rust/src/project/branch_db/branch.rs index 8a3d79b8..b2e749ca 100644 --- a/rust/src/project/branch_db/branch.rs +++ b/rust/src/project/branch_db/branch.rs @@ -18,15 +18,14 @@ impl BranchDb { /// Checks out the initial commit of the main branch automatically. pub async fn create_metadata_doc(&self) -> DocHandle { tracing::info!("Creating new metadata doc..."); - let repo = self.inner.lock().await.repo.clone(); - let username = self.inner.lock().await.username.clone(); + let username = self.username.lock().await.clone(); // Because we always change the checked out ref after creating, we need to lock this in write mode. let r = self.get_checked_out_ref_mut().await; let mut checked_out_ref = r.write().await; // Create new main branch doc - let main_handle = repo.create(Automerge::new()).await.unwrap(); + let main_handle = self.repo.create(Automerge::new()).await.unwrap(); let main_handle_clone = main_handle.clone(); let username_clone = username.clone(); @@ -79,7 +78,7 @@ impl BranchDb { let branches_clone = branches.clone(); // create new branches metadata doc - let metadata_handle = repo.create(Automerge::new()).await.unwrap(); + let metadata_handle = self.repo.create(Automerge::new()).await.unwrap(); let metadata_handle_clone = metadata_handle.clone(); tokio::task::spawn_blocking(move || { metadata_handle.with_document(|d| { @@ -108,4 +107,5 @@ impl BranchDb { .unwrap(); metadata_handle_clone } + } diff --git a/rust/src/project/branch_db/commit.rs b/rust/src/project/branch_db/commit.rs index 56e91c4d..78d9f17a 100644 --- a/rust/src/project/branch_db/commit.rs +++ b/rust/src/project/branch_db/commit.rs @@ -28,7 +28,10 @@ impl BranchDb { is_checking_in: bool, ) -> Option { let Some(branch_handle) = self.get_branch_handle(&ref_.branch).await else { - tracing::error!("Could not commit changes; ref doesn't have an associated branch handle! {:?}", ref_); + tracing::error!( + "Could not commit changes; ref doesn't have an associated branch handle! {:?}", + ref_ + ); return None; }; @@ -36,7 +39,7 @@ impl BranchDb { // Only commit files that have actually changed let files = self.filter_changed_files(ref_, files).await; let count = files.len(); - let username = self.inner.lock().await.username.clone(); + let username = self.username.lock().await.clone(); if (count == 0) { tracing::info!("No actual changes found; not committing."); @@ -223,20 +226,12 @@ impl BranchDb { }) .collect() } - pub async fn create_new_binary_doc(&self, content: Vec) -> DocHandle { tracing::info!("Creating new binary doc..."); - let handle = self - .inner - .lock() - .await - .repo - .create(Automerge::new()) - .await - .unwrap(); + let handle = self.repo.create(Automerge::new()).await.unwrap(); - let username = self.inner.lock().await.username.clone(); + let username = self.username.lock().await.clone(); // we're allowed to transact in the background: nobody needs this to exist yet. let h = handle.clone(); diff --git a/rust/src/project/branch_db/file.rs b/rust/src/project/branch_db/file.rs index 31cae0f1..80c39aa9 100644 --- a/rust/src/project/branch_db/file.rs +++ b/rust/src/project/branch_db/file.rs @@ -194,7 +194,7 @@ impl BranchDb { } async fn get_linked_file(&self, doc_id: &DocumentId) -> Option { - let state = self.inner.lock().await.binary_states.get(doc_id).cloned(); + let state = self.binary_states.lock().await.get(doc_id).cloned(); let Some(handle) = state.and_then(|f| f.doc_handle) else { return None; }; From ce7c7d0cd4f4fcf9e759c7b7ccbe7313b406edc9 Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:39:38 -0800 Subject: [PATCH 09/49] Buggy branch/merge --- .cargo/config.toml | 3 + Cargo.lock | 430 ++++++++++++++++++++- rust/Cargo.toml | 1 + rust/src/helpers/branch.rs | 4 +- rust/src/helpers/tracing.rs | 11 +- rust/src/helpers/utils.rs | 2 +- rust/src/project/branch_db.rs | 10 +- rust/src/project/branch_db/branch.rs | 152 +++++++- rust/src/project/branch_db/commit.rs | 6 +- rust/src/project/branch_db/merge_revert.rs | 154 ++++++++ rust/src/project/branch_db/util.rs | 17 +- rust/src/project/change_ingester.rs | 4 +- rust/src/project/document_watcher.rs | 9 +- rust/src/project/driver.rs | 208 ++++++---- rust/src/project/project.rs | 77 +++- rust/src/project/project_api_impl.rs | 354 ++++++++--------- rust/src/project/sync_automerge_to_fs.rs | 2 +- rust/src/project/sync_fs_to_automerge.rs | 4 +- 18 files changed, 1117 insertions(+), 331 deletions(-) create mode 100644 rust/src/project/branch_db/merge_revert.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index da2da23b..bbe468aa 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,3 +3,6 @@ linker = "arm-linux-gnueabihf-gcc" [target.aarch64-unknown-linux-gnu] linker = "aarch64-linux-gnu-gcc" +# For tokio-console +[build] +rustflags = ["--cfg", "tokio_unstable"] \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 500db02c..05dee1ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + [[package]] name = "async-channel" version = "2.5.0" @@ -110,6 +116,23 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -164,12 +187,61 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "base64" version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "1.3.2" @@ -236,6 +308,12 @@ version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.0" @@ -361,6 +439,46 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console-api" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8599749b6667e2f0c910c1d0dff6901163ff698a52d5a39720f61b5be4b20d3" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tonic-prost", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4915b7d8dd960457a1b6c380114c2944f728e7c65294ab247ae6b6f1f37592" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "hyper-util", + "prost", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.10.1" @@ -607,6 +725,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -832,12 +956,44 @@ dependencies = [ "venial", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "heck" version = "0.4.1" @@ -876,12 +1032,47 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + [[package]] name = "hybrid-array" version = "0.4.5" @@ -891,6 +1082,63 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -1071,6 +1319,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md5" version = "0.8.0" @@ -1083,6 +1337,12 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minicbor" version = "1.1.0" @@ -1103,6 +1363,12 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1140,6 +1406,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a846cbc04412cf509efcd8f3694b114fc700a035fb5a37f21517f9fb019f1ebc" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "notify" version = "8.0.0" @@ -1256,6 +1532,7 @@ dependencies = [ "cargo-post", "cbindgen", "chrono", + "console-subscriber", "futures", "glob", "godot", @@ -1286,6 +1563,32 @@ dependencies = [ "uuid", ] +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1351,6 +1654,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.42" @@ -1609,7 +1944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bab909f3bd4ecf787b3221fd4a8b480f385b3e358e84f7ddb379c0d503aa921" dependencies = [ "automerge", - "base64", + "base64 0.21.7", "bs58", "hex", "minicbor", @@ -1888,6 +2223,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "tempfile" version = "3.23.0" @@ -2010,6 +2351,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.61.2", ] @@ -2141,6 +2483,77 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tonic" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.43" @@ -2256,6 +2669,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae62f7eae5eb549c71b76658648b72cc6111f2d87d24a1e31fa907f4943e3ce" +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "tungstenite" version = "0.27.0" @@ -2362,6 +2781,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e650f458..f9d5d3b6 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -60,6 +60,7 @@ tokio-tungstenite = "0.27.0" async-stream = "0.3.6" tokio-stream = { version = "0.1.18", features = ["sync"] } tokio-util = "0.7.18" +console-subscriber = "0.5.0" [build-dependencies] cbindgen = "^0.27" diff --git a/rust/src/helpers/branch.rs b/rust/src/helpers/branch.rs index 35357ea6..92700ab8 100644 --- a/rust/src/helpers/branch.rs +++ b/rust/src/helpers/branch.rs @@ -39,10 +39,10 @@ pub struct MergeInfo { pub struct Branch { pub name: String, pub id: String, + // TODO: this can just be replaced with HistoryRef if we surgeon that up pub fork_info: Option, pub merge_info: Option, pub created_by: Option, - pub merged_into: Option, pub reverted_to: Option>, } @@ -81,6 +81,4 @@ pub struct BranchState { // These are currently not used by anything, but we want to keep them for later use. #[allow(dead_code)] pub created_by: Option, - #[allow(dead_code)] - pub merged_into: Option, } diff --git a/rust/src/helpers/tracing.rs b/rust/src/helpers/tracing.rs index b445cb62..ffda6232 100644 --- a/rust/src/helpers/tracing.rs +++ b/rust/src/helpers/tracing.rs @@ -25,18 +25,23 @@ impl FormatTime for CompactTime { } static mut M_FILE_WRITER_MUTEX: Option = None; pub fn initialize_tracing() { - let file_appender = tracing_appender::rolling::daily(get_user_dir(), "patchwork.log"); let (non_blocking_file_writer, _guard) = tracing_appender::non_blocking(file_appender); // if the mutex gets dropped, the file writer will be closed, so we need to keep it alive unsafe{M_FILE_WRITER_MUTEX = Some(_guard);} println!("!!! Logging to {:?}/patchwork.log", get_user_dir()); + + let console_layer = console_subscriber::ConsoleLayer::builder() + .with_default_env() + .spawn(); let stdout_layer = tracing_subscriber::fmt::layer() .with_timer(CompactTime) .compact() // .with_span_events(FmtSpan::ENTER | FmtSpan::CLOSE) .with_writer(CustomStdoutWriter::custom_stdout) - .with_filter(EnvFilter::new("trace") + .with_filter(EnvFilter::new("info") + // .add_directive("tokio=trace".parse().unwrap()) + // .add_directive("runtime=trace".parse().unwrap()) .add_directive("patchwork_rust_core=trace".parse().unwrap()) .add_directive("samod=info".parse().unwrap()) .add_directive("samod_core=info".parse().unwrap())); @@ -49,6 +54,8 @@ pub fn initialize_tracing() { .add_directive("samod=info".parse().unwrap()) .add_directive("samod_core=info".parse().unwrap())); if let Err(e) = tracing_subscriber::registry() + // tokio-console + .with(console_layer) // stdout writer .with(stdout_layer) // we want a file writer too diff --git a/rust/src/helpers/utils.rs b/rust/src/helpers/utils.rs index 76ac2bfa..d0bf10bd 100644 --- a/rust/src/helpers/utils.rs +++ b/rust/src/helpers/utils.rs @@ -107,7 +107,7 @@ pub struct CommitMetadata { pub is_setup: Option } -pub(crate) fn commit_with_attribution_and_timestamp(tx: Transaction, metadata: &CommitMetadata) { +pub(crate) fn commit_with_metadata(tx: Transaction, metadata: &CommitMetadata) { let timestamp = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .unwrap() diff --git a/rust/src/project/branch_db.rs b/rust/src/project/branch_db.rs index 804ec0d8..89f0c74f 100644 --- a/rust/src/project/branch_db.rs +++ b/rust/src/project/branch_db.rs @@ -66,7 +66,7 @@ pub struct BranchDb { username: Arc>>, binary_states: Arc>>, branch_states: Arc>>>>, // might be too much locking - metadata_state: Arc>>, + metadata_state: Arc>>, // The checked out ref is the ref that the filesystem is currently synced with. // Has a separate lock because of its importance; it needs to be locked while we're prepping a commit or checking out stuff @@ -102,19 +102,19 @@ impl BranchDb { } /// Get the mutable checked out ref for locking. - pub async fn get_checked_out_ref_mut(&self) -> Arc>> { + pub fn get_checked_out_ref_mut(&self) -> Arc>> { return self.checked_out_ref.clone(); } - pub async fn get_metadata_state(&self) -> Option<(DocumentId, BranchesMetadataDoc)> { + pub async fn get_metadata_state(&self) -> Option<(DocHandle, BranchesMetadataDoc)> { // This is a needlessly expensive operation; we should consider allowing reference introspection via external lockers. // And/or improve clone perf by reducing string usage in BranchesMetadataDoc. self.metadata_state.lock().await.clone() } - pub async fn set_metadata_state(&self, id: DocumentId, state: BranchesMetadataDoc) { + pub async fn set_metadata_state(&self, handle: DocHandle, state: BranchesMetadataDoc) { let mut st = self.metadata_state.lock().await; - *st = Some((id, state)); + *st = Some((handle, state)); } pub async fn has_branch(&self, id: &DocumentId) -> bool { diff --git a/rust/src/project/branch_db/branch.rs b/rust/src/project/branch_db/branch.rs index b2e749ca..90d3f7de 100644 --- a/rust/src/project/branch_db/branch.rs +++ b/rust/src/project/branch_db/branch.rs @@ -1,13 +1,14 @@ use std::collections::HashMap; use automerge::Automerge; -use autosurgeon::reconcile; -use samod::DocHandle; +use autosurgeon::{hydrate, reconcile}; +use samod::{DocHandle, DocumentId}; +use tracing::instrument; use crate::{ helpers::{ - branch::{Branch, BranchesMetadataDoc, GodotProjectDoc}, - utils::{CommitMetadata, commit_with_attribution_and_timestamp}, + branch::{self, Branch, BranchesMetadataDoc, ForkInfo, GodotProjectDoc}, + utils::{CommitMetadata, commit_with_metadata}, }, project::branch_db::{BranchDb, HistoryRef}, }; @@ -19,16 +20,16 @@ impl BranchDb { pub async fn create_metadata_doc(&self) -> DocHandle { tracing::info!("Creating new metadata doc..."); let username = self.username.lock().await.clone(); - + // Because we always change the checked out ref after creating, we need to lock this in write mode. - let r = self.get_checked_out_ref_mut().await; + let r = self.get_checked_out_ref_mut(); let mut checked_out_ref = r.write().await; // Create new main branch doc let main_handle = self.repo.create(Automerge::new()).await.unwrap(); let main_handle_clone = main_handle.clone(); let username_clone = username.clone(); - + let new_heads = tokio::task::spawn_blocking(move || { main_handle_clone.with_document(|d| { let mut tx = d.transaction(); @@ -39,7 +40,7 @@ impl BranchDb { state: HashMap::new(), }, ); - commit_with_attribution_and_timestamp( + commit_with_metadata( tx, &CommitMetadata { username: username_clone, @@ -55,10 +56,10 @@ impl BranchDb { }) .await .unwrap(); - + *checked_out_ref = Some(HistoryRef { branch: main_handle.document_id().clone(), - heads: new_heads + heads: new_heads, }); let main_branch_doc_id = main_handle.document_id().to_string(); @@ -71,7 +72,6 @@ impl BranchDb { fork_info: None, merge_info: None, created_by: username.clone(), - merged_into: None, reverted_to: None, }, )]); @@ -90,7 +90,7 @@ impl BranchDb { branches: branches_clone, }, ); - commit_with_attribution_and_timestamp( + commit_with_metadata( tx, &CommitMetadata { username: username, @@ -107,5 +107,131 @@ impl BranchDb { .unwrap(); metadata_handle_clone } - + + #[instrument(skip_all)] + pub(super) async fn add_branch_to_meta(&self, branch: Branch) { + let meta_handle = { + let meta = self.metadata_state.lock().await; + if meta.is_none() { + tracing::error!("Could not find metadata document!"); + return; + } + meta.as_ref().unwrap().0.clone() + }; + + let username = self.username.lock().await.clone(); + tokio::task::spawn_blocking(move || { + meta_handle.with_document(|d| { + let mut branches_metadata: BranchesMetadataDoc = hydrate(d).unwrap(); + let mut tx = d.transaction(); + branches_metadata.branches.insert(branch.id.clone(), branch); + let _ = reconcile(&mut tx, branches_metadata); + commit_with_metadata( + tx, + &CommitMetadata { + username: username, + branch_id: None, + merge_metadata: None, + reverted_to: None, + changed_files: None, + is_setup: Some(true), + }, + ); + }); + }); + } + + async fn remove_branch_from_meta(&self, branch: DocumentId) { + let meta_handle = { + let meta = self.metadata_state.lock().await; + if meta.is_none() { + tracing::error!("Could not find metadata document!"); + return; + } + meta.as_ref().unwrap().0.clone() + }; + let branch_clone = branch.clone(); + let username = self.username.lock().await.clone(); + tokio::task::spawn_blocking(move || { + meta_handle.with_document(|d| { + let mut tx = d.transaction(); + let mut branches_metadata: BranchesMetadataDoc = hydrate(&mut tx).unwrap(); + branches_metadata.branches.remove(&branch_clone.to_string()); + let _ = reconcile(&mut tx, branches_metadata); + commit_with_metadata( + tx, + &CommitMetadata { + username: username, + branch_id: None, + merge_metadata: None, + reverted_to: None, + changed_files: None, + is_setup: Some(true), + }, + ); + }); + }) + .await + .unwrap(); + } + + // delete branch isn't fully implemented right now deletes are not propagated to the frontend + // right now this is just useful to clean up merge preview branches + #[instrument(skip_all)] + pub async fn delete_branch(&self, branch: &DocumentId) { + self.remove_branch_from_meta(branch.clone()).await; + } + + pub(super) async fn clone_doc(&self, handle: DocHandle) -> DocHandle { + let new_handle = self.repo.create(Automerge::new()).await.unwrap(); + + let new_handle_clone = new_handle.clone(); + tokio::task::spawn_blocking(move || { + handle.with_document(|mut main_d| { + new_handle_clone + .with_document(|d| d.merge(&mut main_d)) + .unwrap(); + }); + }) + .await + .unwrap(); + + return new_handle; + } + + // TODO: This would be more versatile if we gave a HistoryRef instead of a branch. + // That way it might work for reverts too? + pub async fn fork_branch(&self, name: String, source: &DocumentId) -> Option { + tracing::info!("Forking new branch {:?} from source {:?}", name, source); + let Some(source_handle) = self.get_branch_handle(source).await else { + tracing::error!("Couldn't fork branch; existing source branch doesn't exist!"); + return None; + }; + + let Some(latest_ref) = self.get_latest_ref_on_branch(source).await else { + tracing::error!("Couldn't get latest ref on source branch!"); + return None; + }; + let new_handle = self.clone_doc(source_handle).await; + let username = self.username.lock().await.clone(); + let id = new_handle.document_id(); + + self.add_branch_to_meta(Branch { + name: name.clone(), + id: id.to_string(), + fork_info: Some(ForkInfo { + forked_from: source.to_string(), + forked_at: latest_ref + .heads + .into_iter() + .map(|h| h.to_string()) + .collect(), + }), + merge_info: None, + created_by: username, + reverted_to: None, + }) + .await; + Some(id.clone()) + } } diff --git a/rust/src/project/branch_db/commit.rs b/rust/src/project/branch_db/commit.rs index 78d9f17a..f905d8b8 100644 --- a/rust/src/project/branch_db/commit.rs +++ b/rust/src/project/branch_db/commit.rs @@ -9,7 +9,7 @@ use crate::{ helpers::{ doc_utils::SimpleDocReader, utils::{ - ChangeType, ChangedFile, CommitMetadata, commit_with_attribution_and_timestamp, + ChangeType, ChangedFile, CommitMetadata, commit_with_metadata, heads_to_vec_string, }, }, @@ -162,7 +162,7 @@ impl BranchDb { }); } - commit_with_attribution_and_timestamp( + commit_with_metadata( tx, &CommitMetadata { username: username.clone(), @@ -239,7 +239,7 @@ impl BranchDb { h.with_document(|d| { let mut tx = d.transaction(); let _ = tx.put(ROOT, "content", content); - commit_with_attribution_and_timestamp( + commit_with_metadata( tx, &CommitMetadata { username: username, diff --git a/rust/src/project/branch_db/merge_revert.rs b/rust/src/project/branch_db/merge_revert.rs new file mode 100644 index 00000000..ca8cac31 --- /dev/null +++ b/rust/src/project/branch_db/merge_revert.rs @@ -0,0 +1,154 @@ +use automerge::{Automerge, ROOT, transaction::Transactable}; +use autosurgeon::{hydrate, reconcile}; +use samod::DocumentId; +use tracing::instrument; + +use crate::{ + helpers::{ + branch::{Branch, BranchesMetadataDoc, ForkInfo, MergeInfo}, + doc_utils::SimpleDocReader, + utils::{CommitMetadata, MergeMetadata, commit_with_metadata}, + }, + project::branch_db::{BranchDb, branch}, +}; + +impl BranchDb { + #[instrument(skip_all)] + pub async fn create_merge_preview_branch( + &self, + source: &DocumentId, + target: &DocumentId, + ) -> Option { + let Some(source_state) = self.get_branch_state(source).await else { + return None; + }; + let Some(target_state) = self.get_branch_state(target).await else { + return None; + }; + + let (source_state, target_state) = (source_state.lock().await, target_state.lock().await); + + let handle = self.repo.create(Automerge::new()).await.unwrap(); + let handle_clone = handle.clone(); + let source_handle = source_state.doc_handle.clone(); + let target_handle = target_state.doc_handle.clone(); + + tokio::task::spawn_blocking(move || { + source_handle.with_document(|d| { + handle_clone.with_document(|preview_doc| { + let _ = preview_doc.merge(d); + }); + }); + + target_handle.with_document(|d| { + handle_clone.with_document(|preview_doc| { + let _ = preview_doc.merge(d); + }); + }); + }) + .await + .unwrap(); + + let username = self.username.lock().await.clone(); + self.add_branch_to_meta(Branch { + name: format!("{} <- {}", target_state.name, source_state.name), + id: handle.document_id().to_string(), + fork_info: Some(ForkInfo { + forked_from: source.to_string(), + forked_at: source_state + .synced_heads + .iter() + .map(|h| h.to_string()) + .collect(), + }), + merge_info: Some(MergeInfo { + merge_into: target.to_string(), + merge_at: target_state + .synced_heads + .iter() + .map(|h| h.to_string()) + .collect(), + }), + created_by: username.clone(), + reverted_to: None, + }) + .await; + Some(handle.document_id().clone()) + } + + pub async fn merge_branch(&self, source: &DocumentId, target: &DocumentId) { + let Some(source_state) = self.get_branch_state(source).await else { + return; + }; + let Some(target_state) = self.get_branch_state(target).await else { + return; + }; + + let (source_state, target_state) = (source_state.lock().await, target_state.lock().await); + + let source_handle = source_state.doc_handle.clone(); + let target_handle = target_state.doc_handle.clone(); + tokio::task::spawn_blocking(move || { + source_handle.with_document(|d| { + target_handle.with_document(|target| { + let _ = target.merge(d); + }); + }); + }); + + // if the branch has some merge_info we know that it's a merge preview branch + let merge_metadata = if source_state.merge_info.is_some() { + match self + .get_branch_state(&source_state.fork_info.as_ref().unwrap().forked_from) + .await + { + Some(original_state) => { + let original_state = original_state.lock().await; + Some(MergeMetadata { + merged_branch_id: original_state.doc_handle.document_id().clone(), + merged_at_heads: original_state.synced_heads.clone(), + forked_at_heads: original_state + .fork_info + .as_ref() + .unwrap() + .forked_at + .clone(), + }) + } + _ => None, + } + } else { + // todo: implement this case + None + }; + + let username = self.username.lock().await.clone(); + if let Some(merge_metadata) = merge_metadata { + let target = target.clone(); + let target_handle = target_state.doc_handle.clone(); + tokio::task::spawn_blocking(move || { + target_handle.with_document(|d| { + let mut tx = d.transaction(); + + // do a dummy change that we can attach some metadata to + let changed = tx.get_int(&ROOT, "_changed").unwrap_or(0); + let _ = tx.put(ROOT, "_changed", changed + 1); + + commit_with_metadata( + tx, + &CommitMetadata { + username: username.clone(), + branch_id: Some(target.clone()), + merge_metadata: Some(merge_metadata), + reverted_to: None, + changed_files: None, + is_setup: Some(false), + }, + ); + }); + }) + .await + .unwrap(); + } + } +} diff --git a/rust/src/project/branch_db/util.rs b/rust/src/project/branch_db/util.rs index 05df17be..81ebc4cb 100644 --- a/rust/src/project/branch_db/util.rs +++ b/rust/src/project/branch_db/util.rs @@ -1,6 +1,6 @@ use std::{path::PathBuf, str::FromStr}; -use samod::DocumentId; +use samod::{DocHandle, DocumentId}; use tracing::instrument; use crate::project::branch_db::{BranchDb, HistoryRef}; @@ -84,4 +84,19 @@ impl BranchDb { }; Some(state.lock().await.name.clone()) } + + pub async fn get_branch_children(&self, id: &DocumentId) -> Vec { + let states = self.branch_states.lock().await; + let mut result = Vec::new(); + + for (bid, state) in states.iter() { + let state = state.lock().await; + if let Some(fork_info) = &state.fork_info { + if &fork_info.forked_from == id { + result.push(bid.clone()); + } + } + } + result + } } \ No newline at end of file diff --git a/rust/src/project/change_ingester.rs b/rust/src/project/change_ingester.rs index 563e2e47..121d661c 100644 --- a/rust/src/project/change_ingester.rs +++ b/rust/src/project/change_ingester.rs @@ -143,11 +143,9 @@ impl ChangeIngesterInner { } /// Gets the changes from the current branch and returns it. - // TODO (Lilith): This is MISERABLY slow due to the with_document. - // Maybe figure out a way to factor that out. #[tracing::instrument(skip_all)] async fn get_changes(&self) -> Vec { - let checked_out = self.branch_db.get_checked_out_ref_mut().await; + let checked_out = self.branch_db.get_checked_out_ref_mut(); let checked_out = checked_out.read().await; let Some(checked_out) = checked_out.as_ref() else { tracing::info!("Can't get changes; nothing checked out!"); diff --git a/rust/src/project/document_watcher.rs b/rust/src/project/document_watcher.rs index 06b28115..5f66d49f 100644 --- a/rust/src/project/document_watcher.rs +++ b/rust/src/project/document_watcher.rs @@ -144,13 +144,6 @@ impl DocumentWatcherInner { }, is_main: h.document_id().to_string() == meta.main_doc_id, created_by: branch.created_by.clone(), - merged_into: match &branch.merged_into { - Some(merged_into) => match DocumentId::from_str(&merged_into) { - Ok(merged_into) => Some(merged_into), - Err(_) => None, - }, - None => None, - }, revert_info: match &branch.reverted_to { Some(reverted_to) => Some(BranchStateRevertInfo { reverted_to: reverted_to @@ -222,7 +215,7 @@ impl DocumentWatcherInner { .await .unwrap(); self.branch_db - .set_metadata_state(handle.document_id().clone(), meta.clone()) + .set_metadata_state(handle, meta.clone()) .await; // check if there are new branches that haven't loaded yet for (branch_id_str, _) in meta.branches.iter() { diff --git a/rust/src/project/driver.rs b/rust/src/project/driver.rs index 090f15f8..f6ebc3eb 100644 --- a/rust/src/project/driver.rs +++ b/rust/src/project/driver.rs @@ -15,17 +15,17 @@ use automerge::ChangeHash; use futures::channel::oneshot::Cancellation; use futures::{Stream, StreamExt}; use samod::{ConcurrencyConfig, DocHandle, DocumentId, Repo}; -use tokio::select; -use tokio::sync::{Mutex, Notify, broadcast, mpsc, oneshot, watch}; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; -use tokio_stream::wrappers::UnboundedReceiverStream; -use tokio_util::sync::CancellationToken; -use tracing::instrument; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, SystemTime}; +use tokio::select; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio::sync::{Mutex, Notify, broadcast, mpsc, oneshot, watch}; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tokio_util::sync::CancellationToken; +use tracing::instrument; /// The main driver for the project. /// Hooks together all the various controllers. @@ -48,6 +48,9 @@ pub struct DriverInner { safe_to_update_editor: AtomicBool, token: CancellationToken, + // internal synchronization + requested_checkout: Arc>>, + // subtasks repo: Repo, #[allow(unused)] @@ -67,7 +70,8 @@ pub struct DriverInner { impl Drop for Driver { fn drop(&mut self) { self.token.cancel(); - self.repo.stop(); + // just use the default executor for this one I think? + futures::executor::block_on(self.repo.stop()); } } @@ -172,7 +176,7 @@ impl Driver { let peer_watcher = Arc::new(PeerWatcher::new(repo.clone())); let sync_automerge_to_fs = SyncAutomergeToFileSystem::new(branch_db.clone()); let sync_fs_to_automerge = SyncFileSystemToAutomerge::new(branch_db.clone()); - + let metadata_handle = match &metadata_id { // If we're expecting an existing ID, try and fetch it. Some(id) => { @@ -182,15 +186,12 @@ impl Driver { handle } // If we need to make a new ID, make a doc and check in the initial state of the filesystem. - None => { - branch_db.create_metadata_doc().await - } + None => branch_db.create_metadata_doc().await, }; // The document watcher will auto-ingest the provided metadata handle. let document_watcher = DocumentWatcher::new(repo.clone(), branch_db.clone(), metadata_handle).await; - // If this is a new project (i.e. we earlier made a metadata doc), check in the files. // This has to go after the document watcher ingests the metadata doc, of course. @@ -212,25 +213,26 @@ impl Driver { let this = Some(Driver { file_changes_rx, inner: Arc::new(DriverInner { - token: token.clone(), main_thread_block, file_changes_tx, ref_tx, safe_to_update_editor: AtomicBool::new(false), + token: token.clone(), + requested_checkout: Arc::new(Mutex::new(None)), repo: repo.clone(), connection, branch_db, peer_watcher, - document_watcher, change_ingester, + document_watcher, sync_automerge_to_fs, sync_fs_to_automerge, // differ, }), repo, - token + token, }); - + // Spawn off the sync task let inner_clone = this.as_ref().unwrap().inner.clone(); tokio::spawn(async move { @@ -267,6 +269,15 @@ impl Driver { } } + /// Request the sync task to checkout the latest ref on a branch the next opportunity. + /// This will only work once Godot is safe to update. + // TODO (Lilith): This is broken for immediately checked-out branches. + // We should instead allow request_checkout to poll for other branches that haven't loaded in yet. + pub async fn request_checkout(&self, branch: &DocumentId) { + let mut req = self.inner.requested_checkout.lock().await; + *req = Some(branch.clone()); + } + async fn get_metadata_handle( repo: &Repo, metadata_id: &DocumentId, @@ -308,7 +319,7 @@ impl Driver { ); return None; } - Err(e) => { + Err(_) => { tracing::error!( "Can't start the driver; the repo was immediately stopped!" ); @@ -319,31 +330,50 @@ impl Driver { } } - pub fn merge_branch( - &mut self, - source_branch_doc_id: DocumentId, - target_branch_doc_id: DocumentId, - ) { - // TODO + pub async fn fork_branch(&self, name: String, branch: &DocumentId) { + if let Some(id) = self.inner.branch_db.fork_branch(name, branch).await { + self.request_checkout(&id).await; + } } - pub fn create_merge_preview_branch_between( - &mut self, - source_branch_doc_id: DocumentId, - target_branch_doc_id: DocumentId, - ) { - // TODO (Lilith) + pub async fn merge_branch(&self, source: &DocumentId, target: &DocumentId) { + self.inner.branch_db.merge_branch(source, target).await; + self.inner.branch_db.delete_branch(source).await; + self.request_checkout(target).await; } - pub fn create_revert_preview_branch_for( - &mut self, - branch_doc_id: DocumentId, - revert_to: Vec, - ) { - // TODO (Lilith) + pub async fn discard_current_branch(&self) { + let Some(checked_out_ref) = self.get_checked_out_ref().await else { + return; + }; + + let Some(branch_state) = self.get_branch_state(&checked_out_ref.branch).await else { + return; + }; + + let Some(fork_info) = &branch_state.fork_info else { + return; + }; + self.inner + .branch_db + .delete_branch(&branch_state.doc_handle.document_id().clone()) + .await; + + self.request_checkout(&fork_info.forked_from).await; } - pub fn delete_branch(&mut self, branch_doc_id: DocumentId) { + pub async fn create_merge_preview_branch(&self, source: &DocumentId, target: &DocumentId) { + if let Some(id) = self + .inner + .branch_db + .create_merge_preview_branch(source, target) + .await + { + self.request_checkout(&id).await; + } + } + + pub fn create_revert_preview_branch(&mut self, ref_: &HistoryRef) { // TODO (Lilith) } @@ -375,7 +405,7 @@ impl Driver { .branch_db .get_metadata_state() .await - .map(|(id, _)| id) + .map(|(handle, _)| handle.document_id().clone()) } pub async fn get_main_branch(&self) -> Option { @@ -400,13 +430,19 @@ impl Driver { Some(state.lock().await.clone()) } + pub async fn get_branch_children(&self, id: &DocumentId) -> Vec { + self.inner.branch_db.get_branch_children(id).await + } + pub async fn get_checked_out_ref(&self) -> Option { - let checked_out_ref = self.inner.branch_db.get_checked_out_ref_mut().await; + let checked_out_ref = self.inner.branch_db.get_checked_out_ref_mut(); return checked_out_ref.read().await.clone(); } pub fn set_safe_to_update_editor(&self, safe: bool) { - self.inner.safe_to_update_editor.store(safe, Ordering::Relaxed); + self.inner + .safe_to_update_editor + .store(safe, Ordering::Relaxed); } // awkward @@ -445,7 +481,13 @@ impl DriverInner { #[instrument(skip_all)] async fn sync(&self) { - let old_checked_out_ref = self.branch_db.get_checked_out_ref_mut().await.read().await.clone(); + tracing::trace!("Syncing..."); + let old_checked_out_ref = self + .branch_db + .get_checked_out_ref_mut() + .read() + .await + .clone(); // Ensure we block the main thread inside of Rust while checking out a ref. // Very important to not allow Godot to explode while we're writing files! { @@ -463,12 +505,55 @@ impl DriverInner { self.change_ingester.request_ingestion(); } - let new_checked_out_ref = self.branch_db.get_checked_out_ref_mut().await.read().await.clone(); + let new_checked_out_ref = self + .branch_db + .get_checked_out_ref_mut() + .read() + .await + .clone(); // If we've changed branches, send the new checked out ref. - if new_checked_out_ref.as_ref().map(|r| &r.branch) != old_checked_out_ref.as_ref().map(|r| &r.branch) { + if new_checked_out_ref.as_ref().map(|r| &r.branch) + != old_checked_out_ref.as_ref().map(|r| &r.branch) + { self.ref_tx.send(new_checked_out_ref).unwrap(); } + tracing::trace!("Done with sync."); + } + + async fn get_ref_for_sync(&self) -> Option { + let mut requested_checkout = self.requested_checkout.lock().await; + + // The logic here: + // - If we have a requested checkout that is valid, use that, and clear it + // - If the requested checkout is invalid or empty, use the branch from the currently checked out ref + // - If we don't have anything currently checked out, default to main. + let req_branch = requested_checkout.clone(); + if let Some(requested_branch) = req_branch { + if let Some(latest) = self + .branch_db + .get_latest_ref_on_branch(&requested_branch) + .await + { + requested_checkout.take(); // clear it + return Some(latest); + } + } + + let current_ref = self.branch_db.get_checked_out_ref_mut(); + let current_ref = current_ref.read().await; + if let Some(current_ref) = current_ref.clone() { + return Some(current_ref); + } + if let Some(main_branch) = self.branch_db.get_main_branch().await { + if let Some(ref_) = self.branch_db.get_latest_ref_on_branch(&main_branch).await { + return Some(ref_); + } + } + tracing::error!( + "No metadata doc checked out, or otherwise couldn't get main branch. Skipping checkout!" + ); + return None; } /// If our current ref is out-of-date, try and check out a new ref. @@ -482,39 +567,10 @@ impl DriverInner { // find any actual changes. // Maybe that's OK, we need to profile to see if it's a problem. - let goal_ref = { - let current_ref_lock = self.branch_db.get_checked_out_ref_mut().await; - let current_ref_guard = current_ref_lock.read().await; - - let branch = match current_ref_guard.as_ref() { - Some(r) => r.branch.clone(), - None => { - // If we didn't find a current ref, nothing is checked out. - // Let's fix that by checking out the main branch. - let Some(main_branch) = self.branch_db.get_main_branch().await else { - tracing::error!( - "No metadata doc checked out, or otherwise couldn't get main branch. Skipping sync!" - ); - return Vec::new(); - }; - main_branch - } - }; - let Some(goal_ref) = self.branch_db.get_latest_ref_on_branch(&branch).await - else { - tracing::error!("Couldn't get the goal ref for branch {}", branch); - return Vec::new(); - }; - goal_ref - // guard dropped here - }; - // TODO (Lilith): minor problem, the checked out ref could change between these lines when the guard is dropped. - // That said, uhhhh I think that's fine? We're already syncing, so other sync methods shouldn't affect it. - // If we're doing some branch edits, like a merge or revert preview or something, that could mean we - // never checkout our desired ref... but we need to rethink this for branch swapping anyways. - // For branch swapping, we either need to checkout a ref elsewhere during a merge or something and prevent sync. - // Or, we could set a request_checkout tokio::Watch here on the driver that will check out a branch at the next opportunity. - self.sync_automerge_to_fs.checkout_ref(goal_ref).await + if let Some(goal_ref) = self.get_ref_for_sync().await { + self.sync_automerge_to_fs.checkout_ref(goal_ref).await + } else { + Vec::new() + } } - } diff --git a/rust/src/project/project.rs b/rust/src/project/project.rs index e7991e6b..974f957d 100644 --- a/rust/src/project/project.rs +++ b/rust/src/project/project.rs @@ -1,7 +1,10 @@ use crate::diff::differ::ProjectDiff; use crate::fs::file_utils::FileSystemEvent; +use crate::helpers::branch::BranchState; use crate::helpers::utils::{CommitInfo, summarize_changes}; -use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkConfigAccessor, PatchworkEditorAccessor}; +use crate::interop::godot_accessors::{ + EditorFilesystemAccessor, PatchworkConfigAccessor, PatchworkEditorAccessor, +}; use crate::project::branch_db::HistoryRef; use crate::project::driver::Driver; use crate::project::main_thread_block::MainThreadBlock; @@ -9,13 +12,14 @@ use crate::project::project_api::ChangeViewModel; use automerge::ChangeHash; use futures::future::join_all; use samod::DocumentId; -use tokio::sync::{Mutex, watch}; +use tracing::instrument; use std::cell::RefCell; use std::path::PathBuf; use std::sync::Arc; use std::time::SystemTime; use std::{collections::HashMap, str::FromStr}; use tokio::runtime::Runtime; +use tokio::sync::{Mutex, MutexGuard, OwnedMutexGuard, watch}; /// Manages the state and operations of a Patchwork project within Godot. /// Its API is exposed to GDScript via the GodotProject struct. @@ -80,8 +84,8 @@ impl Project { fn ingest_changes(&mut self, changes: Vec) { tracing::info!("Ingesting changes..."); - self.history.clear(); - self.changes.clear(); + self.history.clear(); + self.changes.clear(); // Consume changes into self.changes for change in changes { @@ -89,7 +93,7 @@ impl Project { self.changes.insert(change.hash, change); } } - + pub fn get_cached_diff(&self, before: HistoryRef, after: HistoryRef) -> ProjectDiff { self.diff_cache .borrow_mut() @@ -102,7 +106,7 @@ impl Project { self.diff_cache.borrow_mut().clear(); } - // Do not run this on anything except the main thread! + // Do not run this on anything except the main thread! pub fn safe_to_update_godot() -> bool { return !(EditorFilesystemAccessor::is_scanning() || PatchworkEditorAccessor::is_editor_importing() @@ -191,7 +195,7 @@ impl Project { tracing::error!("Could not start the driver!"); return; } - + let driver = self.driver.blocking_lock(); self.changes_rx = Some(driver.as_ref().unwrap().get_changes_rx()); self.checked_out_ref_rx = Some(driver.as_ref().unwrap().get_ref_rx()); @@ -201,23 +205,63 @@ impl Project { self.driver.blocking_lock().take(); } + pub(super) fn get_checked_out_branch_state(&self) -> Option { + self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return None; + } + let branch_state = match driver.as_ref().unwrap().get_checked_out_ref().await { + Some(id) => driver.as_ref().unwrap().get_branch_state(&id.branch).await, + None => None, + }; + branch_state.clone() + }) + } + + /// Jank utility function to lock on the driver and run on a different thread. + /// Allows us to easily block on async code when we need the driver. + pub(super) fn with_driver_blocking(&self, f: F) -> R + where + F: FnOnce(OwnedMutexGuard>) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + R: Send + 'static, + { + let driver = self.driver.clone(); + + self.runtime + .block_on(self.runtime.spawn(async move { + let driver = driver.lock_owned().await; + f(driver).await + })) + .unwrap() + } + + #[instrument(skip_all)] pub fn process(&mut self, _delta: f64) -> (Vec, Vec) { + tracing::trace!("Running project process..."); let fs_changes = { let mut driver_guard = self.driver.blocking_lock(); if driver_guard.is_none() { return (Vec::new(), Vec::new()); } // Run the blocking sync - driver_guard.as_ref().unwrap().set_safe_to_update_editor(Self::safe_to_update_godot()); + driver_guard + .as_ref() + .unwrap() + .set_safe_to_update_editor(Self::safe_to_update_godot()); let block = self.main_thread_block.clone(); - self.runtime.block_on(self.runtime.spawn(async move { - block.checkpoint().await; - })).unwrap(); + tracing::trace!("Blocking for dependents..."); + self.runtime + .block_on(self.runtime.spawn(async move { + block.checkpoint().await; + })) + .unwrap(); + tracing::trace!("Done blocking."); // Consume any modified files to send to Godot driver_guard.as_mut().unwrap().get_filesystem_changes() }; - + let mut signals = Vec::new(); // Ingest changes if the driver produced a new changeset @@ -227,25 +271,26 @@ impl Project { rx.mark_unchanged(); signals.push(GodotProjectSignal::ChangesIngested); Some(rx.borrow().clone()) - } - else { + } else { None } }; - + if let Some(changes) = changes { self.ingest_changes(changes); } // Check to see if we need to produce a CheckedOutBranch signal - let rx = self.checked_out_ref_rx.as_ref().unwrap(); + let rx = self.checked_out_ref_rx.as_mut().unwrap(); if rx.has_changed().unwrap_or(false) { signals.push(GodotProjectSignal::CheckedOutBranch); + rx.mark_unchanged(); } // TODO (Lilith): VERY IMPORTANT, set the patchwork config branch ID here!!! // So that we save the branch ID for future checkouts. + tracing::trace!("Done with process."); (fs_changes, signals) } } diff --git a/rust/src/project/project_api_impl.rs b/rust/src/project/project_api_impl.rs index 6f3f3419..1efbce16 100644 --- a/rust/src/project/project_api_impl.rs +++ b/rust/src/project/project_api_impl.rs @@ -23,23 +23,21 @@ use crate::{ // TODO: Ideally this is actually a child of a new project submodule... // that's so that it doesn't need pub(super) to acess private fields of // itself. + // TODO (Lilith): Figure out if there's a reasonable way to reduce blocking in this file. +// In general I kind of hate this, but I guess a sync/async divide is never going to look pretty. impl ProjectViewModel for Project { fn has_project(&self) -> bool { self.driver.blocking_lock().is_some() } fn get_project_id(&self) -> Option { - let driver = self.driver.clone(); - self.runtime - .block_on(self.runtime.spawn(async move { - let driver = driver.lock().await; - if driver.is_none() { - return None; - } - driver.as_ref().unwrap().get_metadata_doc().await - })) - .unwrap() + self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return None; + } + driver.as_ref().unwrap().get_metadata_doc().await + }) } fn new_project(&mut self) { @@ -77,40 +75,41 @@ impl ProjectViewModel for Project { fn set_user_name(&self, name: String) { PatchworkConfigAccessor::set_user_value("user_name", &name); - let driver = self.driver.clone(); - self.runtime - .block_on(self.runtime.spawn(async move { - let mut driver = driver.lock().await; - if driver.is_none() { - return; - } - driver.as_mut().unwrap().set_username(Some(name)).await - })) - .unwrap(); + self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return; + } + driver.as_ref().unwrap().set_username(Some(name)).await + }); } fn can_create_merge_preview_branch(&self) -> bool { - // TODO (Lilith): implement - return false; - // match self.get_checked_out_branch_state() { - // Some(branch_state) => !branch_state.is_main, - // _ => false, - // } + match self.get_checked_out_branch_state() { + Some(branch_state) => !branch_state.is_main, + _ => false, + } } fn create_merge_preview_branch(&mut self) { - // TODO (Lilith): implement - return; - // let Some(checked_out_branch) = self.get_checked_out_branch_state() else { - // return; - // }; - // let Some(fork_info) = &checked_out_branch.fork_info else { - // return; - // }; - // self.create_merge_preview_branch_between( - // checked_out_branch.doc_handle.document_id().clone(), - // fork_info.forked_from.clone(), - // ); + let Some(checked_out_branch) = self.get_checked_out_branch_state() else { + return; + }; + let Some(fork_info) = &checked_out_branch.fork_info else { + return; + }; + + let source = checked_out_branch.doc_handle.document_id().clone(); + let target = fork_info.forked_from.clone(); + self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return; + } + driver + .as_ref() + .unwrap() + .create_merge_preview_branch(&source, &target) + .await; + }); } fn can_create_revert_preview_branch(&self, head: ChangeHash) -> bool { @@ -150,73 +149,92 @@ impl ProjectViewModel for Project { } fn is_merge_preview_branch_active(&self) -> bool { - // TODO (Lilith): implement - return false; - // let branch_state = self.get_checked_out_branch_state(); - // match branch_state { - // Some(state) => state.merge_info.is_some(), - // _ => false, - // } + let branch_state = self.get_checked_out_branch_state(); + match branch_state { + Some(state) => state.merge_info.is_some(), + _ => false, + } } fn is_safe_to_merge(&self) -> bool { - // TODO (Lilith): implement - return false; - // let Some(current_branch) = self.get_checked_out_branch_state() else { - // return false; - // }; - // let Some(merge_info) = current_branch.merge_info.as_ref() else { - // return false; - // }; - // let Some(fork_info) = current_branch.fork_info.as_ref() else { - // return false; - // }; + let Some(current_branch) = self.get_checked_out_branch_state() else { + return false; + }; + let Some(merge_info) = current_branch.merge_info.as_ref() else { + return false; + }; + let Some(fork_info) = current_branch.fork_info.as_ref() else { + return false; + }; - // let source_branch = self.branch_states.get(&fork_info.forked_from); - // let dest_branch = self.branch_states.get(&merge_info.merge_into); + let forked_from = fork_info.forked_from.clone(); + let merge_into = merge_info.merge_into.clone(); + let Some((source_branch, dest_branch)) = self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return None; + } + let source_branch = driver + .as_ref() + .unwrap() + .get_branch_state(&forked_from) + .await; + let dest_branch = driver.as_ref().unwrap().get_branch_state(&merge_into).await; + Some((source_branch, dest_branch)) + }) else { + return false; + }; - // let Some(dest_branch) = dest_branch else { - // return false; - // }; + let Some(dest_branch) = dest_branch else { + return false; + }; - // source_branch.is_some_and(|s| { - // s.fork_info - // .as_ref() - // .is_some_and(|i| i.forked_at == dest_branch.synced_heads) - // }) + source_branch.is_some_and(|s| { + s.fork_info + .as_ref() + .is_some_and(|i| i.forked_at == dest_branch.synced_heads) + }) } fn confirm_preview_branch(&mut self) { - // TODO (Lilith): implement - return; - // let Some(branch_state) = self.get_checked_out_branch_state().cloned() else { - // return; - // }; - // let Some(fork_info) = &branch_state.fork_info else { - // return; - // }; - // if let Some(revert_info) = &branch_state.revert_info { - // self.delete_branch(branch_state.doc_handle.document_id().clone()); - // self.checkout_branch(fork_info.forked_from.clone()); - // self.revert_to_heads(revert_info.reverted_to.clone()); - // } else if let Some(merge_info) = branch_state.merge_info { - // self.merge_branch( - // branch_state.doc_handle.document_id().clone(), - // merge_info.merge_into, - // ) - // } + let Some(branch_state) = self.get_checked_out_branch_state() else { + return; + }; + let Some(fork_info) = &branch_state.fork_info else { + return; + }; + if let Some(revert_info) = &branch_state.revert_info { + // TODO (Lilith): Implement + // self.delete_branch(branch_state.doc_handle.document_id().clone()); + // self.checkout_branch(fork_info.forked_from.clone()); + // self.revert_to_heads(revert_info.reverted_to.clone()); + } else if let Some(merge_info) = branch_state.merge_info { + let source = branch_state.doc_handle.document_id().clone(); + let target = merge_info.merge_into; + self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return; + } + driver + .as_ref() + .unwrap() + .merge_branch(&source, &target) + .await; + }); + } } fn discard_preview_branch(&mut self) { - // TODO (Lilith): implement - return; - // let Some(branch_state) = self.get_checked_out_branch_state().cloned() else { - // return; - // }; - // let Some(fork_info) = &branch_state.fork_info else { - // return; - // }; - // self.delete_branch(branch_state.doc_handle.document_id().clone()); - // self.checkout_branch(fork_info.forked_from.clone()); + let Some(branch_state) = self.get_checked_out_branch_state() else { + return; + }; + let Some(fork_info) = &branch_state.fork_info else { + return; + }; + self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return; + } + driver.as_ref().unwrap().discard_current_branch().await; + }); } fn get_branch_history(&self) -> Vec { @@ -302,48 +320,35 @@ impl ProjectViewModel for Project { } fn get_branch(&self, id: &DocumentId) -> Option> { - let driver = self.driver.clone(); let id = id.clone(); - let state = self - .runtime - .block_on(self.runtime.spawn(async move { - let driver = driver.lock().await; - if driver.is_none() { - return None; - } - driver.as_ref().unwrap().get_branch_state(&id).await - })) - .unwrap(); - let Some(state) = state else { + + let Some((state, mut children)) = self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return None; + } + let Some(state) = driver.as_ref().unwrap().get_branch_state(&id).await else { + return None; + }; + let children = driver.as_ref().unwrap().get_branch_children(&id).await; + Some((state, children)) + }) else { return None; }; - // TODO (Lilith): implement - // let mut children = self - // .branch_states - // .values() - // .filter(|b| { - // b.fork_info - // .as_ref() - // .is_some_and(|i| i.forked_from == id.clone()) - // }) - // .map(|b| b.doc_handle.document_id().clone()) - // .collect::>(); - - // children.sort_by(|a, b| { - // let a_state = self.branch_states.get(&a); - // let b_state = self.branch_states.get(&b); - // let Some(a_state) = a_state else { - // return std::cmp::Ordering::Less; - // }; - // let Some(b_state) = b_state else { - // return std::cmp::Ordering::Greater; - // }; - // a_state - // .name - // .to_lowercase() - // .cmp(&b_state.name.to_lowercase()) - // }); + children.sort_by(|a, b| { + let a_state = self.get_branch(a); + let b_state = self.get_branch(b); + let Some(a_state) = a_state else { + return std::cmp::Ordering::Less; + }; + let Some(b_state) = b_state else { + return std::cmp::Ordering::Greater; + }; + a_state + .get_name() + .to_lowercase() + .cmp(&b_state.get_name().to_lowercase()) + }); Some(BranchWrapper { state: state.clone(), @@ -382,74 +387,31 @@ impl ProjectViewModel for Project { self.get_branch(&id.branch) } - #[instrument(skip(self), fields(name = ?name), level = tracing::Level::INFO)] fn create_branch(&mut self, name: String) { - // TODO (Lilith): implement - return; - // println!(""); - // tracing::info!("******** CREATE BRANCH"); - // println!(""); - // let source_branch_doc_id = match &self.get_checked_out_branch_state() { - // Some(branch_state) => branch_state.doc_handle.document_id(), - // None => { - // panic!("couldn't create branch, no checked out branch"); - // } - // }; - - // self.driver_input_tx - // .unbounded_send(InputEvent::CreateBranch { - // name, - // source_branch_doc_id: source_branch_doc_id.clone(), - // }) - // .unwrap(); - - // // TODO: do we want to set this? or let _process set it? - // self.checked_out_branch_state = - // CheckedOutBranchState::NothingCheckedOut(Some(source_branch_doc_id.clone())); - // // self.checked_out_branch_state = CheckedOutBranchState::NothingCheckedOut(None); + let Some(branch_state) = self.get_checked_out_branch_state() else { + return; + }; + self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return; + } + driver + .as_ref() + .unwrap() + .fork_branch(name, branch_state.doc_handle.document_id()).await; + }); } fn checkout_branch(&mut self, branch_doc_id: DocumentId) { - // TODO (Lilith): implement - return; - // let current_branch = match &self.checked_out_branch_state { - // CheckedOutBranchState::CheckedOut(doc_id, _) => Some(doc_id.clone()), - // CheckedOutBranchState::CheckingOut(doc_id, _) => { - // tracing::error!( - // "**@#%@#%!@#%#@!*** CHECKING OUT BRANCH WHILE STILL CHECKING OUT?!?!?! {:?}", - // doc_id - // ); - // Some(doc_id.clone()) - // } - // CheckedOutBranchState::NothingCheckedOut(current_branch_id) => { - // tracing::warn!("Checking out a branch while not checked out on any branch????"); - // current_branch_id.clone() - // } - // }; - // let target_branch_state = match self.branch_states.get(&branch_doc_id) { - // Some(branch_state) => branch_state, - // None => panic!("couldn't checkout branch, branch doc id not found"), - // }; - // println!(""); - // tracing::debug!("******** CHECKOUT: {:?}\n", target_branch_state.name); - // println!(""); - - // if target_branch_state.synced_heads - // == target_branch_state - // .doc_handle - // .with_document(|d| d.get_heads()) - // { - // self.checked_out_branch_state = - // CheckedOutBranchState::CheckedOut(branch_doc_id.clone(), current_branch.clone()); - // self.just_checked_out_new_branch = true; - // } else { - // tracing::debug!( - // "checked out branch {:?} has unsynced heads", - // target_branch_state.name - // ); - // self.checked_out_branch_state = - // CheckedOutBranchState::CheckingOut(branch_doc_id.clone(), current_branch.clone()); - // } + self.with_driver_blocking(|driver| async move { + if driver.is_none() { + return; + } + driver + .as_ref() + .unwrap() + .request_checkout(&branch_doc_id).await; + }); } fn get_change(&self, hash: ChangeHash) -> Option<&impl ChangeViewModel> { diff --git a/rust/src/project/sync_automerge_to_fs.rs b/rust/src/project/sync_automerge_to_fs.rs index 1264b73a..9da94f43 100644 --- a/rust/src/project/sync_automerge_to_fs.rs +++ b/rust/src/project/sync_automerge_to_fs.rs @@ -29,7 +29,7 @@ impl SyncAutomergeToFileSystem { #[instrument(skip_all)] pub async fn checkout_ref(&self, goal_ref: HistoryRef) -> Vec { // Ensure that there's no way anything can grab the ref while we're trying to write it - let r = self.branch_db.get_checked_out_ref_mut().await; + let r = self.branch_db.get_checked_out_ref_mut(); let mut checked_out_ref = r.write().await; if checked_out_ref.as_ref().is_some_and(|r| r == &goal_ref) { diff --git a/rust/src/project/sync_fs_to_automerge.rs b/rust/src/project/sync_fs_to_automerge.rs index 038fe435..668a2c85 100644 --- a/rust/src/project/sync_fs_to_automerge.rs +++ b/rust/src/project/sync_fs_to_automerge.rs @@ -79,7 +79,7 @@ impl SyncFileSystemToAutomerge { #[instrument(skip_all)] pub async fn commit(&self) -> bool { // Because we always change the checked out ref after committing, we need to lock this in write mode. - let r = self.branch_db.get_checked_out_ref_mut().await; + let r = self.branch_db.get_checked_out_ref_mut(); let mut checked_out_ref = r.write().await; let mut pending_changes = self.pending_changes.lock().await; @@ -126,7 +126,7 @@ impl SyncFileSystemToAutomerge { /// Makes the commit on the currently checked-out branch, and checks out the new heads. pub async fn checkin(&self) { // Because we always change the checked out ref after committing, we need to lock this in write mode. - let r = self.branch_db.get_checked_out_ref_mut().await; + let r = self.branch_db.get_checked_out_ref_mut(); let mut checked_out_ref = r.write().await; if checked_out_ref.is_none() { From 7b7431afc712e19146a872828ad8afcac6901fab Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Tue, 27 Jan 2026 19:53:49 -0800 Subject: [PATCH 10/49] tokio-console support and deadlock fix --- rust/Cargo.toml | 2 +- rust/src/helpers/tracing.rs | 1 + rust/src/helpers/utils.rs | 24 ++++ rust/src/project/branch_db.rs | 21 ++- rust/src/project/branch_db/file.rs | 6 +- rust/src/project/branch_db/merge_revert.rs | 31 ++--- rust/src/project/branch_db/util.rs | 32 +++-- rust/src/project/change_ingester.rs | 12 +- rust/src/project/connection.rs | 4 +- rust/src/project/document_watcher.rs | 16 +-- rust/src/project/driver.rs | 14 +- rust/src/project/peer_watcher.rs | 4 +- rust/src/project/project.rs | 20 +-- rust/src/project/project_api_impl.rs | 146 ++++++--------------- rust/src/project/sync_fs_to_automerge.rs | 5 +- 15 files changed, 143 insertions(+), 195 deletions(-) diff --git a/rust/Cargo.toml b/rust/Cargo.toml index f9d5d3b6..a10f5ab7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -24,7 +24,7 @@ automerge = "0.7.2" autosurgeon = "0.10.1" futures = "0.3.31" godot = "0.4.3" -tokio = { version = "1.49.0", features = ["full"] } +tokio = { version = "1.49.0", features = ["full", "tracing"] } # tracing only works with unstable; we need it to log task names tracing-subscriber = { version = "0.3.19", features = ["env-filter", "json"] } tree-sitter = "0.25.4" tree-sitter-godot-resource = { git = "https://github.com/nikitalita/tree-sitter-godot-resource.git", branch = "fix-string-names" } diff --git a/rust/src/helpers/tracing.rs b/rust/src/helpers/tracing.rs index ffda6232..7836474b 100644 --- a/rust/src/helpers/tracing.rs +++ b/rust/src/helpers/tracing.rs @@ -31,6 +31,7 @@ pub fn initialize_tracing() { unsafe{M_FILE_WRITER_MUTEX = Some(_guard);} println!("!!! Logging to {:?}/patchwork.log", get_user_dir()); + // TODO (Lilith): Maybe guard this behind a debug flag let console_layer = console_subscriber::ConsoleLayer::builder() .with_default_env() .spawn(); diff --git a/rust/src/helpers/utils.rs b/rust/src/helpers/utils.rs index d0bf10bd..556c1989 100644 --- a/rust/src/helpers/utils.rs +++ b/rust/src/helpers/utils.rs @@ -9,6 +9,8 @@ use automerge::{ use samod::{DocHandle, DocumentId}; use chrono::{DateTime, Local}; use serde::{Deserialize, Serialize}; +use tokio::{runtime::{self, Runtime}, task::JoinHandle}; +use tracing::Instrument; #[inline(always)] pub(crate) fn get_automerge_doc_diff(doc: &Automerge, old_heads: &[ChangeHash], new_heads: &[ChangeHash]) -> Vec { @@ -287,3 +289,25 @@ pub fn exact_human_readable_timestamp(timestamp: i64) -> String { let datetime : DateTime = DateTime::from(dt.unwrap()); return datetime.format("%Y-%m-%d %H:%M:%S").to_string(); } + +pub fn spawn_named(name: &str, future: F) -> JoinHandle where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + let span = tracing::info_span!("task", name = name); + tokio::task::Builder::new() + .name(name) + .spawn(future.instrument(span)) + .expect(&format!("Something went wrong trying to build the task {name}.")) +} + +pub fn spawn_named_on(name: &str, runtime: &runtime::Handle, future: F) -> JoinHandle where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + let span = tracing::info_span!("task", name = name); + tokio::task::Builder::new() + .name(name) + .spawn_on(future.instrument(span), runtime) + .expect(&format!("Something went wrong trying to build the task {name}.")) +} \ No newline at end of file diff --git a/rust/src/project/branch_db.rs b/rust/src/project/branch_db.rs index 89f0c74f..6692fc8f 100644 --- a/rust/src/project/branch_db.rs +++ b/rust/src/project/branch_db.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, path::PathBuf, sync::Arc, }; @@ -102,6 +102,7 @@ impl BranchDb { } /// Get the mutable checked out ref for locking. + /// TODO (Lilith): This smells kind of nasty, maybe don't expose this... but how else to ensure we don't step on toes? pub fn get_checked_out_ref_mut(&self) -> Arc>> { return self.checked_out_ref.clone(); } @@ -132,18 +133,12 @@ impl BranchDb { .or_insert_with(|| Arc::new(Mutex::new(f()))); } - // This exposes inner BranchState objects via Arc. This is important because we use branch states all over the place. - // Alternatively we could provide a read-only view in a closure, or clone them. - // We do need to be a little careful about locks though. - pub async fn get_branch_state(&self, id: &DocumentId) -> Option>> { - let st = self.branch_states.lock().await; - st.get(id).cloned() - } - - pub async fn get_branch_handle(&self, id: &DocumentId) -> Option { - let Some(state) = self.get_branch_state(id).await else { - return None; + pub async fn set_linked_docs_for_branch(&self, id: &DocumentId, linked_docs: HashSet) { + let states = self.branch_states.lock().await; + let Some(state) = states.get(id) else { + return; }; - Some(state.lock().await.doc_handle.clone()) + let mut state = state.lock().await; + state.linked_doc_ids = linked_docs; } } diff --git a/rust/src/project/branch_db/file.rs b/rust/src/project/branch_db/file.rs index 80c39aa9..c41ae72f 100644 --- a/rust/src/project/branch_db/file.rs +++ b/rust/src/project/branch_db/file.rs @@ -220,14 +220,10 @@ impl BranchDb { filters: &HashSet, ) -> Option> { tracing::info!("Getting files at ref {:?}", desired_ref); - let Some(branch_state) = self.get_branch_state(&desired_ref.branch).await else { - return None; - }; - let mut files = HashMap::new(); let mut linked_doc_ids = Vec::new(); - let doc_handle = branch_state.lock().await.doc_handle.clone(); + let doc_handle = self.get_branch_handle(&desired_ref.branch).await?; let filters = filters.clone(); let desired_ref = desired_ref.clone(); let (mut files, linked_doc_ids) = tokio::task::spawn_blocking(move || { diff --git a/rust/src/project/branch_db/merge_revert.rs b/rust/src/project/branch_db/merge_revert.rs index ca8cac31..37e26bfb 100644 --- a/rust/src/project/branch_db/merge_revert.rs +++ b/rust/src/project/branch_db/merge_revert.rs @@ -19,19 +19,17 @@ impl BranchDb { source: &DocumentId, target: &DocumentId, ) -> Option { - let Some(source_state) = self.get_branch_state(source).await else { - return None; - }; - let Some(target_state) = self.get_branch_state(target).await else { - return None; - }; - - let (source_state, target_state) = (source_state.lock().await, target_state.lock().await); + // Not getting the branch state so we don't gotta clone, honestly that was probably simpler though + let source_name = self.get_branch_name(source).await?; + let target_name = self.get_branch_name(target).await?; + let source_handle = self.get_branch_handle(source).await?; + let target_handle = self.get_branch_handle(target).await?; + + let source_ref = self.get_latest_ref_on_branch(source).await?; + let target_ref = self.get_latest_ref_on_branch(target).await?; let handle = self.repo.create(Automerge::new()).await.unwrap(); let handle_clone = handle.clone(); - let source_handle = source_state.doc_handle.clone(); - let target_handle = target_state.doc_handle.clone(); tokio::task::spawn_blocking(move || { source_handle.with_document(|d| { @@ -51,20 +49,20 @@ impl BranchDb { let username = self.username.lock().await.clone(); self.add_branch_to_meta(Branch { - name: format!("{} <- {}", target_state.name, source_state.name), + name: format!("{} <- {}", target_name, source_name), id: handle.document_id().to_string(), fork_info: Some(ForkInfo { forked_from: source.to_string(), - forked_at: source_state - .synced_heads + forked_at: source_ref + .heads .iter() .map(|h| h.to_string()) .collect(), }), merge_info: Some(MergeInfo { merge_into: target.to_string(), - merge_at: target_state - .synced_heads + merge_at: target_ref + .heads .iter() .map(|h| h.to_string()) .collect(), @@ -84,8 +82,6 @@ impl BranchDb { return; }; - let (source_state, target_state) = (source_state.lock().await, target_state.lock().await); - let source_handle = source_state.doc_handle.clone(); let target_handle = target_state.doc_handle.clone(); tokio::task::spawn_blocking(move || { @@ -103,7 +99,6 @@ impl BranchDb { .await { Some(original_state) => { - let original_state = original_state.lock().await; Some(MergeMetadata { merged_branch_id: original_state.doc_handle.document_id().clone(), merged_at_heads: original_state.synced_heads.clone(), diff --git a/rust/src/project/branch_db/util.rs b/rust/src/project/branch_db/util.rs index 81ebc4cb..3a961d4b 100644 --- a/rust/src/project/branch_db/util.rs +++ b/rust/src/project/branch_db/util.rs @@ -3,7 +3,7 @@ use std::{path::PathBuf, str::FromStr}; use samod::{DocHandle, DocumentId}; use tracing::instrument; -use crate::project::branch_db::{BranchDb, HistoryRef}; +use crate::{helpers::branch::BranchState, project::branch_db::{BranchDb, HistoryRef}}; // Utility methods for working with [BranchDb]. impl BranchDb { @@ -38,13 +38,7 @@ impl BranchDb { // Maybe that's enough? Get DocumentWatcher to do it? Then we remove the with_doc call here. #[instrument(skip_all)] pub async fn get_latest_ref_on_branch(&self, branch: &DocumentId) -> Option { - let state = self.get_branch_state(branch).await; - let Some(state) = state else { - tracing::error!("Couldn't get latest ref on branch; branch state not loaded!"); - return None; - }; - let state = state.lock().await; - let handle = state.doc_handle.clone(); + let handle = self.get_branch_handle(branch).await?; let heads = tokio::task::spawn_blocking(move || handle.with_document(|d| d.get_heads())).await.unwrap(); Some(HistoryRef { @@ -79,10 +73,22 @@ impl BranchDb { } pub async fn get_branch_name(&self, id: &DocumentId) -> Option { - let Some(state) = self.get_branch_state(id).await else { - return None; - }; - Some(state.lock().await.name.clone()) + let states = self.branch_states.lock().await; + Some(states.get(id)?.lock().await.name.clone()) + } + + // This is not ideal -- I'd prefer not to clone unless necessary. + // However, we NEVER want to expose our internal BranchState mutexes. + // That could cause deadlocks if they acquired a branch state and later tried to call any branch info method on branch_db. + // Callers should preferentially use other getter methods. + pub async fn get_branch_state(&self, id: &DocumentId) -> Option { + let states = self.branch_states.lock().await; + Some(states.get(id)?.lock().await.clone()) + } + + pub async fn get_branch_handle(&self, id: &DocumentId) -> Option { + let states = self.branch_states.lock().await; + Some(states.get(id)?.lock().await.doc_handle.clone()) } pub async fn get_branch_children(&self, id: &DocumentId) -> Vec { @@ -90,7 +96,7 @@ impl BranchDb { let mut result = Vec::new(); for (bid, state) in states.iter() { - let state = state.lock().await; + let state: tokio::sync::MutexGuard<'_, crate::helpers::branch::BranchState> = state.lock().await; if let Some(fork_info) = &state.fork_info { if &fork_info.forked_from == id { result.push(bid.clone()); diff --git a/rust/src/project/change_ingester.rs b/rust/src/project/change_ingester.rs index 121d661c..f4aa16ae 100644 --- a/rust/src/project/change_ingester.rs +++ b/rust/src/project/change_ingester.rs @@ -11,7 +11,7 @@ use tokio::{ use tokio_util::sync::CancellationToken; use crate::{ - helpers::utils::{CommitInfo, CommitMetadata, summarize_changes}, + helpers::utils::{CommitInfo, CommitMetadata, spawn_named, summarize_changes}, project::{branch_db::BranchDb, peer_watcher::PeerWatcher, project_api::ChangeViewModel}, }; @@ -52,7 +52,7 @@ impl ChangeIngester { }); let inner_clone = inner.clone(); - tokio::spawn(async move { + spawn_named("Change ingester", async move { let stream = inner_clone.peer_watcher.subscribe(); tokio::pin!(stream); @@ -152,14 +152,10 @@ impl ChangeIngesterInner { return Vec::new(); }; - let Some(branch_state) = self.branch_db.get_branch_state(&checked_out.branch).await else { - tracing::info!("Can't get the checked out branch state; something must be wrong"); + let Some(handle) = self.branch_db.get_branch_handle(&checked_out.branch).await else { + tracing::info!("Can't get the checked out branch handle; something must be wrong"); return Vec::new(); }; - - // TODO: we probably don't need to lock the branch state for this whole method - let branch_state = branch_state.lock().await; - let handle = branch_state.doc_handle.clone(); let doc_id = handle.document_id(); let last_acked_heads = self diff --git a/rust/src/project/connection.rs b/rust/src/project/connection.rs index 73c7a97a..ba7e02f1 100644 --- a/rust/src/project/connection.rs +++ b/rust/src/project/connection.rs @@ -8,6 +8,8 @@ use tokio::{ use tokio_stream::wrappers::BroadcastStream; use tokio_util::sync::CancellationToken; +use crate::helpers::utils::spawn_named; + #[derive(Debug, Clone)] enum ConnectionStoppedReason { TcpConnectionError(String), @@ -79,7 +81,7 @@ impl RemoteConnection { }); let inner_clone = inner.clone(); - tokio::spawn(async move { + spawn_named("Remote connection", async move { inner_clone.retry_connection().await; }); diff --git a/rust/src/project/document_watcher.rs b/rust/src/project/document_watcher.rs index 5f66d49f..6eb1308c 100644 --- a/rust/src/project/document_watcher.rs +++ b/rust/src/project/document_watcher.rs @@ -11,7 +11,7 @@ use crate::{ BranchesMetadataDoc, }, doc_utils::SimpleDocReader, - utils::parse_automerge_url, + utils::{parse_automerge_url, spawn_named}, }, project::branch_db::BranchDb, }; @@ -58,7 +58,7 @@ impl DocumentWatcher { .await; // track changes for future ingests - tokio::spawn(async move { + spawn_named("Metadata tracker", async move { inner_clone.track_metadata_document(metadata_handle).await; }); @@ -193,13 +193,7 @@ impl DocumentWatcherInner { .await .unwrap(); - let branch_state_mutex = self - .branch_db - .get_branch_state(&handle.document_id()) - .await - .unwrap(); - let mut branch_state = branch_state_mutex.lock().await; - branch_state.linked_doc_ids = linked_docs.values().cloned().collect(); + self.branch_db.set_linked_docs_for_branch(handle.document_id(), linked_docs.values().cloned().collect()); } #[tracing::instrument(skip_all)] @@ -232,7 +226,9 @@ impl DocumentWatcherInner { self.ingest_branch_document(handle.clone()).await; // Track the document let this = self.clone(); - tokio::spawn(async move { this.track_branch_document(handle).await }); + spawn_named(&format!("Document tracker: {:?}", branch_id), async move { + this.track_branch_document(handle).await + }); } } } diff --git a/rust/src/project/driver.rs b/rust/src/project/driver.rs index f6ebc3eb..e3db3131 100644 --- a/rust/src/project/driver.rs +++ b/rust/src/project/driver.rs @@ -1,7 +1,7 @@ use crate::diff::differ::ProjectDiff; use crate::fs::file_utils::FileSystemEvent; use crate::helpers::branch::{self, BranchState}; -use crate::helpers::utils::{CommitInfo, CommitMetadata}; +use crate::helpers::utils::{CommitInfo, CommitMetadata, spawn_named}; use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkEditorAccessor}; use crate::project::branch_db::{BranchDb, HistoryRef}; use crate::project::change_ingester::ChangeIngester; @@ -235,7 +235,7 @@ impl Driver { // Spawn off the sync task let inner_clone = this.as_ref().unwrap().inner.clone(); - tokio::spawn(async move { + spawn_named("Sync", async move { inner_clone.sync_main().await; }); this @@ -417,17 +417,11 @@ impl Driver { } pub async fn get_branch_name(&self, id: &DocumentId) -> Option { - let Some(state) = self.inner.branch_db.get_branch_state(id).await else { - return None; - }; - Some(state.lock().await.name.clone()) + self.inner.branch_db.get_branch_name(id).await } pub async fn get_branch_state(&self, id: &DocumentId) -> Option { - let Some(state) = self.inner.branch_db.get_branch_state(id).await else { - return None; - }; - Some(state.lock().await.clone()) + self.inner.branch_db.get_branch_state(id).await } pub async fn get_branch_children(&self, id: &DocumentId) -> Vec { diff --git a/rust/src/project/peer_watcher.rs b/rust/src/project/peer_watcher.rs index c8b64622..9c77fe8d 100644 --- a/rust/src/project/peer_watcher.rs +++ b/rust/src/project/peer_watcher.rs @@ -4,6 +4,8 @@ use tokio::{select, sync::watch}; use tokio_stream::wrappers::WatchStream; use tokio_util::sync::CancellationToken; +use crate::helpers::utils::spawn_named; + #[derive(Debug)] pub struct PeerWatcher { server_info_tx: watch::Sender>, @@ -23,7 +25,7 @@ impl PeerWatcher { let repo_handle_clone = repo_handle.clone(); let token = CancellationToken::new(); let token_clone = token.clone(); - tokio::spawn(async move { + spawn_named("Peer watcher", async move { let (_, stream) = repo_handle_clone.connected_peers(); tokio::pin!(stream); loop { diff --git a/rust/src/project/project.rs b/rust/src/project/project.rs index 974f957d..e8512f1e 100644 --- a/rust/src/project/project.rs +++ b/rust/src/project/project.rs @@ -1,7 +1,7 @@ use crate::diff::differ::ProjectDiff; use crate::fs::file_utils::FileSystemEvent; use crate::helpers::branch::BranchState; -use crate::helpers::utils::{CommitInfo, summarize_changes}; +use crate::helpers::utils::{CommitInfo, spawn_named, spawn_named_on, summarize_changes}; use crate::interop::godot_accessors::{ EditorFilesystemAccessor, PatchworkConfigAccessor, PatchworkEditorAccessor, }; @@ -185,7 +185,7 @@ impl Project { .runtime .block_on( // I think it's correct to spawn this on a different task explicitly, because block_on runs the future on the current thread, not a worker thread. - self.runtime.spawn(async move { + spawn_named_on("Create driver", self.runtime.handle(), async move { Driver::new(block, server_url, project_dir, storage_dir, metadata_id).await }), ) @@ -205,8 +205,9 @@ impl Project { self.driver.blocking_lock().take(); } + // common utility function within this class pub(super) fn get_checked_out_branch_state(&self) -> Option { - self.with_driver_blocking(|driver| async move { + self.with_driver_blocking("Get checked out branch state", |driver| async move { if driver.is_none() { return None; } @@ -220,18 +221,21 @@ impl Project { /// Jank utility function to lock on the driver and run on a different thread. /// Allows us to easily block on async code when we need the driver. - pub(super) fn with_driver_blocking(&self, f: F) -> R + pub(super) fn with_driver_blocking(&self, name: &str, f: F) -> R where F: FnOnce(OwnedMutexGuard>) -> Fut + Send + 'static, Fut: Future + Send + 'static, R: Send + 'static, { let driver = self.driver.clone(); - + let name_clone = name.to_string(); self.runtime - .block_on(self.runtime.spawn(async move { + .block_on(spawn_named_on(name, self.runtime.handle(), async move { + tracing::trace!("Starting block on {name_clone}..."); let driver = driver.lock_owned().await; - f(driver).await + let res =f(driver).await; + tracing::trace!("Finishing block on {name_clone}!"); + return res; })) .unwrap() } @@ -252,7 +256,7 @@ impl Project { let block = self.main_thread_block.clone(); tracing::trace!("Blocking for dependents..."); self.runtime - .block_on(self.runtime.spawn(async move { + .block_on(spawn_named_on("Blocking guard", self.runtime.handle(), async move { block.checkpoint().await; })) .unwrap(); diff --git a/rust/src/project/project_api_impl.rs b/rust/src/project/project_api_impl.rs index 1efbce16..4a081492 100644 --- a/rust/src/project/project_api_impl.rs +++ b/rust/src/project/project_api_impl.rs @@ -8,7 +8,7 @@ use crate::{ diff::differ::ProjectDiff, helpers::utils::{ BranchWrapper, CommitInfo, DiffWrapper, exact_human_readable_timestamp, - human_readable_timestamp, + human_readable_timestamp, spawn_named, spawn_named_on, }, interop::godot_accessors::PatchworkConfigAccessor, project::{ @@ -32,11 +32,8 @@ impl ProjectViewModel for Project { } fn get_project_id(&self) -> Option { - self.with_driver_blocking(|driver| async move { - if driver.is_none() { - return None; - } - driver.as_ref().unwrap().get_metadata_doc().await + self.with_driver_blocking("Get project ID", |driver| async move { + driver.as_ref()?.get_metadata_doc().await }) } @@ -75,11 +72,9 @@ impl ProjectViewModel for Project { fn set_user_name(&self, name: String) { PatchworkConfigAccessor::set_user_value("user_name", &name); - self.with_driver_blocking(|driver| async move { - if driver.is_none() { - return; - } - driver.as_ref().unwrap().set_username(Some(name)).await + self.with_driver_blocking("Set username", |driver| async move { + driver.as_ref()?.set_username(Some(name)).await; + Some(()) }); } @@ -100,15 +95,12 @@ impl ProjectViewModel for Project { let source = checked_out_branch.doc_handle.document_id().clone(); let target = fork_info.forked_from.clone(); - self.with_driver_blocking(|driver| async move { - if driver.is_none() { - return; - } + self.with_driver_blocking("Create merge preview branch", |driver| async move { driver - .as_ref() - .unwrap() + .as_ref()? .create_merge_preview_branch(&source, &target) .await; + Some(()) }); } @@ -169,13 +161,9 @@ impl ProjectViewModel for Project { let forked_from = fork_info.forked_from.clone(); let merge_into = merge_info.merge_into.clone(); - let Some((source_branch, dest_branch)) = self.with_driver_blocking(|driver| async move { - if driver.is_none() { - return None; - } + let Some((source_branch, dest_branch)) = self.with_driver_blocking("Is safe to merge", |driver| async move { let source_branch = driver - .as_ref() - .unwrap() + .as_ref()? .get_branch_state(&forked_from) .await; let dest_branch = driver.as_ref().unwrap().get_branch_state(&merge_into).await; @@ -210,15 +198,12 @@ impl ProjectViewModel for Project { } else if let Some(merge_info) = branch_state.merge_info { let source = branch_state.doc_handle.document_id().clone(); let target = merge_info.merge_into; - self.with_driver_blocking(|driver| async move { - if driver.is_none() { - return; - } + self.with_driver_blocking("Confirm merge preview branch", |driver| async move { driver - .as_ref() - .unwrap() + .as_ref()? .merge_branch(&source, &target) .await; + Some(()) }); } } @@ -229,11 +214,9 @@ impl ProjectViewModel for Project { let Some(fork_info) = &branch_state.fork_info else { return; }; - self.with_driver_blocking(|driver| async move { - if driver.is_none() { - return; - } - driver.as_ref().unwrap().discard_current_branch().await; + self.with_driver_blocking("Discard preview branch", |driver| async move { + driver.as_ref()?.discard_current_branch().await; + Some(()) }); } @@ -322,14 +305,13 @@ impl ProjectViewModel for Project { fn get_branch(&self, id: &DocumentId) -> Option> { let id = id.clone(); - let Some((state, mut children)) = self.with_driver_blocking(|driver| async move { - if driver.is_none() { - return None; - } - let Some(state) = driver.as_ref().unwrap().get_branch_state(&id).await else { + let Some((state, mut children)) = self.with_driver_blocking("Get branch", |driver| async move { + tracing::trace!("Getting branch state..."); + let Some(state) = driver.as_ref()?.get_branch_state(&id).await else { return None; }; - let children = driver.as_ref().unwrap().get_branch_children(&id).await; + tracing::trace!("Getting branch children..."); + let children = driver.as_ref()?.get_branch_children(&id).await; Some((state, children)) }) else { return None; @@ -358,32 +340,16 @@ impl ProjectViewModel for Project { } fn get_main_branch(&self) -> Option { - let driver = self.driver.clone(); - let id = self - .runtime - .block_on(self.runtime.spawn(async move { - let driver = driver.lock().await; - if driver.is_none() { - return None; - } - driver.as_ref().unwrap().get_main_branch().await - })) - .unwrap()?; + let id = self.with_driver_blocking("Get main branch", |driver| async move { + driver.as_ref()?.get_main_branch().await + })?; self.get_branch(&id) } fn get_checked_out_branch(&self) -> Option { - let driver = self.driver.clone(); - let id = self - .runtime - .block_on(self.runtime.spawn(async move { - let driver = driver.lock().await; - if driver.is_none() { - return None; - } - driver.as_ref().unwrap().get_checked_out_ref().await - })) - .unwrap()?; + let id = self.with_driver_blocking("Get checked out branch", |driver| async move { + driver.as_ref()?.get_checked_out_ref().await + })?; self.get_branch(&id.branch) } @@ -391,26 +357,20 @@ impl ProjectViewModel for Project { let Some(branch_state) = self.get_checked_out_branch_state() else { return; }; - self.with_driver_blocking(|driver| async move { - if driver.is_none() { - return; - } + self.with_driver_blocking("Create branch", |driver| async move { driver - .as_ref() - .unwrap() + .as_ref()? .fork_branch(name, branch_state.doc_handle.document_id()).await; + Some(()) }); } fn checkout_branch(&mut self, branch_doc_id: DocumentId) { - self.with_driver_blocking(|driver| async move { - if driver.is_none() { - return; - } + self.with_driver_blocking("Checkout branch", |driver| async move { driver - .as_ref() - .unwrap() + .as_ref()? .request_checkout(&branch_doc_id).await; + Some(()) }); } @@ -421,21 +381,13 @@ impl ProjectViewModel for Project { fn get_default_diff(&self) -> Option { let heads_before; let heads_after; - let driver = self.driver.clone(); - let branch_state = self - .runtime - .block_on(self.runtime.spawn(async move { - let driver = driver.lock().await; - if driver.is_none() { - return None; - } - driver - .as_ref() - .unwrap() - .get_branch_state(&driver.as_ref().unwrap().get_checked_out_ref().await?.branch) - .await - })) - .unwrap()?; + + let branch_state = self.with_driver_blocking("Get default diff", |driver| async move { + driver + .as_ref()? + .get_branch_state(&driver.as_ref().unwrap().get_checked_out_ref().await?.branch) + .await + })?; // There is no default diff for the main branch! if branch_state.is_main { @@ -521,21 +473,7 @@ impl ProjectViewModel for Project { } } - let driver = self.driver.clone(); - let branch_state = self - .runtime - .block_on(self.runtime.spawn(async move { - let driver = driver.lock().await; - if driver.is_none() { - return None; - } - driver - .as_ref() - .unwrap() - .get_branch_state(&driver.as_ref().unwrap().get_checked_out_ref().await?.branch) - .await - })) - .unwrap()?; + let branch_state = self.get_checked_out_branch_state()?; if let Some(prev_hash) = prev_hash { heads_before = vec![prev_hash]; diff --git a/rust/src/project/sync_fs_to_automerge.rs b/rust/src/project/sync_fs_to_automerge.rs index 668a2c85..b430e5ba 100644 --- a/rust/src/project/sync_fs_to_automerge.rs +++ b/rust/src/project/sync_fs_to_automerge.rs @@ -6,8 +6,7 @@ use tokio_util::sync::CancellationToken; use tracing::instrument; use crate::{ - fs::{file_utils::FileContent, file_utils::FileSystemEvent}, - project::{branch_db::BranchDb, fs_watcher::FileSystemWatcher}, + fs::file_utils::{FileContent, FileSystemEvent}, helpers::utils::spawn_named, project::{branch_db::BranchDb, fs_watcher::FileSystemWatcher} }; /// Tracks changes using [FileSystemWatcher], handles the changes, and tracks them as pending. @@ -40,7 +39,7 @@ impl SyncFileSystemToAutomerge { let token_clone = token.clone(); // TODO (Lilith): stick this on a method on an Inner struct like the rest - tokio::spawn(async move { + spawn_named("Sync FS to Automerge", async move { let changes = FileSystemWatcher::start_watching( branch_db_clone.get_project_dir().clone(), branch_db_clone.clone(), From e16e6e8c90ba654ae67c2e468c99343200d75cf5 Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Wed, 28 Jan 2026 05:13:55 -0800 Subject: [PATCH 11/49] make tokio-console optional --- .cargo/config.toml | 5 +- Developers.md | 1 + justfile | 32 ++++--- rust/Cargo.toml | 7 +- rust/src/helpers.rs | 1 + rust/src/helpers/spawn_utils.rs | 54 +++++++++++ rust/src/helpers/tracing.rs | 111 ++++++++++++----------- rust/src/helpers/utils.rs | 22 ----- rust/src/project/change_ingester.rs | 2 +- rust/src/project/connection.rs | 2 +- rust/src/project/document_watcher.rs | 4 +- rust/src/project/driver.rs | 3 +- rust/src/project/peer_watcher.rs | 2 +- rust/src/project/project.rs | 3 +- rust/src/project/project_api_impl.rs | 2 +- rust/src/project/sync_fs_to_automerge.rs | 2 +- 16 files changed, 151 insertions(+), 102 deletions(-) create mode 100644 rust/src/helpers/spawn_utils.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index bbe468aa..acedb593 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,7 +2,4 @@ [target.armv7-unknown-linux-gnueabihf] linker = "arm-linux-gnueabihf-gcc" [target.aarch64-unknown-linux-gnu] -linker = "aarch64-linux-gnu-gcc" -# For tokio-console -[build] -rustflags = ["--cfg", "tokio_unstable"] \ No newline at end of file +linker = "aarch64-linux-gnu-gcc" \ No newline at end of file diff --git a/Developers.md b/Developers.md index b402731a..9bbfee11 100644 --- a/Developers.md +++ b/Developers.md @@ -45,6 +45,7 @@ project=[moddable-platformer|threadbare] # launch threadbare or moddable-platfor patchwork_profile=[release|debug] # whether we should build the rust code with release or debug configuration godot_profile=[release|debug|sani] # whether we should build Godot with release, debug, or sani configuration server_url= # force embed a server URL into the project. By default, just keeps whatever server URL is already configured in the project. +tracing_support=[none|tokio-console] # allows a tokio-console to be connected at the default port for debugging ``` #### Using Visual Studio Code diff --git a/justfile b/justfile index 8783e2dd..489fa33f 100644 --- a/justfile +++ b/justfile @@ -134,12 +134,19 @@ build-godot profile: _link-godot fi # Build the Rust plugin binaries. -_build-plugin architecture profile: - cargo build --profile="{{profile}}" --target="{{architecture}}" +_build-plugin architecture profile tracing_support: + #!/usr/bin/env sh + if [[ {{tracing_support}} = "tokio-console" ]] ; then + export RUSTFLAGS="--cfg tokio_unstable" + cargo build --profile="{{profile}}" --target="{{architecture}}" --features "tokio-console" + else + cargo build --profile="{{profile}}" --target="{{architecture}}" + fi # Build the multi-arch target for MacOS. [parallel] -_build-plugin-all-macos profile: (_build-plugin "aarch64-apple-darwin" profile) (_build-plugin "x86_64-apple-darwin" profile) _make-plugin-dir +_build-plugin-all-macos profile tracing_support: (_build-plugin "aarch64-apple-darwin" profile tracing_support) \ + (_build-plugin "x86_64-apple-darwin" profile tracing_support) _make-plugin-dir mkdir -p build/patchwork/bin # Copy the entire macos directory to get the Resources framework directory @@ -153,7 +160,7 @@ _build-plugin-all-macos profile: (_build-plugin "aarch64-apple-darwin" profile) # TODO: Perhaps sign here instead of in github actions? [parallel] -_build-plugin-single-arch architecture profile: (_build-plugin architecture profile) _make-plugin-dir +_build-plugin-single-arch architecture profile tracing_support: (_build-plugin architecture profile tracing_support) _make-plugin-dir #!/usr/bin/env sh # set -euo pipefail mkdir -p build/patchwork/bin @@ -235,11 +242,12 @@ _configure-patchwork: _make-plugin-dir # Build the plugin and output it to the plugin build dir. For MacOS multi-arch, use architecture=all-apple-darwin to build all architectures. [parallel] [arg('profile', pattern='release|debug')] -build-patchwork profile architecture=(default_arch): _configure-patchwork _link-public +[arg('tracing_support', pattern='none|tokio-console')] +build-patchwork profile architecture=(default_arch) tracing_support="none": _configure-patchwork _link-public #!/usr/bin/env sh # set -euxo pipefail if [[ "{{architecture}}" = "all-apple-darwin" ]] ; then - just _build-plugin-all-macos "{{profile}}" + just _build-plugin-all-macos "{{profile}}" "{{tracing_support}}" exit 0 fi @@ -248,7 +256,7 @@ build-patchwork profile architecture=(default_arch): _configure-patchwork _link- profile="release_debug" fi - just _build-plugin-single-arch "{{architecture}}" "$profile" + just _build-plugin-single-arch "{{architecture}}" "$profile" "{{tracing_support}}" # Reset the Godot repository, removing the linked module and resetting the repo state. clean-godot: @@ -348,16 +356,18 @@ _write-url project url: (_link-project project) [arg('project', pattern='moddable-platformer|threadbare')] [arg('patchwork_profile', pattern='release|debug')] [arg('godot_profile', pattern='release|debug|sani')] -prepare project="moddable-platformer" patchwork_profile="release" godot_profile="release" server_url="":\ - (_link-project project) (build-godot godot_profile) (build-patchwork patchwork_profile) (_write-url project server_url) +[arg('tracing_support', pattern='none|tokio-console')] +prepare project="moddable-platformer" patchwork_profile="release" godot_profile="release" server_url="" tracing_support="none": \ + (_link-project project) (build-godot godot_profile) (build-patchwork patchwork_profile default_arch tracing_support) (_write-url project server_url) # Launch a project with Godot. Available projects are threadbare, moddable-platformer. [arg('project', pattern='moddable-platformer|threadbare')] [arg('patchwork_profile', pattern='release|debug')] [arg('godot_profile', pattern='release|debug|sani')] -launch project="moddable-platformer" patchwork_profile="release" godot_profile="release" server_url="": \ - (prepare project patchwork_profile godot_profile server_url) +[arg('tracing_support', pattern='none|tokio-console')] +launch project="moddable-platformer" patchwork_profile="release" godot_profile="release" server_url="" tracing_support="none": \ + (prepare project patchwork_profile godot_profile server_url tracing_support) #!/usr/bin/env sh # set -euxo pipefail diff --git a/rust/Cargo.toml b/rust/Cargo.toml index a10f5ab7..17f4eff1 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -16,15 +16,17 @@ bench = false doc = false [features] -# default = ["automerge_0_6"] +default = [] automerge_0_6 = [] +tokio-console = ["dep:console-subscriber", "tokio/tracing"] [dependencies] automerge = "0.7.2" autosurgeon = "0.10.1" futures = "0.3.31" godot = "0.4.3" -tokio = { version = "1.49.0", features = ["full", "tracing"] } # tracing only works with unstable; we need it to log task names +tokio = { version = "1.49.0", features = ["full"] } +console-subscriber = { version = "0.5.0", optional = true } tracing-subscriber = { version = "0.3.19", features = ["env-filter", "json"] } tree-sitter = "0.25.4" tree-sitter-godot-resource = { git = "https://github.com/nikitalita/tree-sitter-godot-resource.git", branch = "fix-string-names" } @@ -60,7 +62,6 @@ tokio-tungstenite = "0.27.0" async-stream = "0.3.6" tokio-stream = { version = "0.1.18", features = ["sync"] } tokio-util = "0.7.18" -console-subscriber = "0.5.0" [build-dependencies] cbindgen = "^0.27" diff --git a/rust/src/helpers.rs b/rust/src/helpers.rs index 1c3b3f11..a62e59eb 100644 --- a/rust/src/helpers.rs +++ b/rust/src/helpers.rs @@ -2,3 +2,4 @@ pub mod tracing; pub mod doc_utils; pub mod utils; pub mod branch; +pub mod spawn_utils; \ No newline at end of file diff --git a/rust/src/helpers/spawn_utils.rs b/rust/src/helpers/spawn_utils.rs new file mode 100644 index 00000000..ae4d01be --- /dev/null +++ b/rust/src/helpers/spawn_utils.rs @@ -0,0 +1,54 @@ +use tokio::runtime::Handle; +use tokio::task::JoinHandle; + +#[cfg(feature = "tokio-console")] +pub fn spawn_named(name: &str, future: F) -> JoinHandle +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + use tracing::Instrument; + let span = tracing::info_span!("task"); + tokio::task::Builder::new() + .name(name) + .spawn(future.instrument(span)) + .expect(&format!( + "Something went wrong trying to build the task {name}." + )) +} + +#[cfg(not(feature = "tokio-console"))] +#[warn(unused_variables)] +pub fn spawn_named(name: &str, future: F) -> JoinHandle +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + tokio::spawn(future) +} + +#[cfg(feature = "tokio-console")] +pub fn spawn_named_on(name: &str, runtime: &Handle, future: F) -> JoinHandle +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + use tracing::Instrument; + + let span = tracing::info_span!("task"); + tokio::task::Builder::new() + .name(name) + .spawn_on(future.instrument(span), runtime) + .expect(&format!( + "Something went wrong trying to build the task {name}." + )) +} + +#[cfg(not(feature = "tokio-console"))] +pub fn spawn_named_on(name: &str, runtime: &Handle, future: F) -> JoinHandle +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + runtime.spawn(future) +} diff --git a/rust/src/helpers/tracing.rs b/rust/src/helpers/tracing.rs index 7836474b..b01163bb 100644 --- a/rust/src/helpers/tracing.rs +++ b/rust/src/helpers/tracing.rs @@ -2,9 +2,14 @@ use std::fmt; use std::io::Write; use godot::classes::ProjectSettings; -use tracing_appender::non_blocking::WorkerGuard; -use tracing_subscriber::{EnvFilter, Layer, fmt::{format::{FmtSpan, Writer}, time::FormatTime}, layer::SubscriberExt, util::SubscriberInitExt}; use godot::obj::Singleton; +use tracing_appender::non_blocking::WorkerGuard; +use tracing_subscriber::{ + EnvFilter, Layer, + fmt::{format::Writer, time::FormatTime}, + layer::SubscriberExt, + util::SubscriberInitExt, +}; fn get_user_dir() -> String { let user_dir = ProjectSettings::singleton() @@ -16,60 +21,67 @@ fn get_user_dir() -> String { struct CompactTime; impl FormatTime for CompactTime { fn format_time(&self, w: &mut Writer<'_>) -> Result<(), std::fmt::Error> { - write!( - w, - "{}", - TimeNoDate::from(std::time::SystemTime::now()) - ) + write!(w, "{}", TimeNoDate::from(std::time::SystemTime::now())) } } static mut M_FILE_WRITER_MUTEX: Option = None; pub fn initialize_tracing() { let file_appender = tracing_appender::rolling::daily(get_user_dir(), "patchwork.log"); let (non_blocking_file_writer, _guard) = tracing_appender::non_blocking(file_appender); - // if the mutex gets dropped, the file writer will be closed, so we need to keep it alive - unsafe{M_FILE_WRITER_MUTEX = Some(_guard);} + // if the mutex gets dropped, the file writer will be closed, so we need to keep it alive + unsafe { + M_FILE_WRITER_MUTEX = Some(_guard); + } println!("!!! Logging to {:?}/patchwork.log", get_user_dir()); - // TODO (Lilith): Maybe guard this behind a debug flag - let console_layer = console_subscriber::ConsoleLayer::builder() - .with_default_env() - .spawn(); let stdout_layer = tracing_subscriber::fmt::layer() .with_timer(CompactTime) .compact() // .with_span_events(FmtSpan::ENTER | FmtSpan::CLOSE) .with_writer(CustomStdoutWriter::custom_stdout) - .with_filter(EnvFilter::new("info") - // .add_directive("tokio=trace".parse().unwrap()) - // .add_directive("runtime=trace".parse().unwrap()) - .add_directive("patchwork_rust_core=trace".parse().unwrap()) - .add_directive("samod=info".parse().unwrap()) - .add_directive("samod_core=info".parse().unwrap())); + .with_filter( + EnvFilter::new("info") + // .add_directive("tokio=trace".parse().unwrap()) + // .add_directive("runtime=trace".parse().unwrap()) + .add_directive("patchwork_rust_core=debug".parse().unwrap()) + .add_directive("samod=info".parse().unwrap()) + .add_directive("samod_core=info".parse().unwrap()), + ); let file_layer = tracing_subscriber::fmt::layer() .with_line_number(true) - .with_ansi(false) + .with_ansi(false) .with_writer(non_blocking_file_writer.clone()) - .with_filter(EnvFilter::new("info") - .add_directive("patchwork_rust_core=trace".parse().unwrap()) - .add_directive("samod=info".parse().unwrap()) - .add_directive("samod_core=info".parse().unwrap())); - if let Err(e) = tracing_subscriber::registry() - // tokio-console - .with(console_layer) - // stdout writer + .with_filter( + EnvFilter::new("info") + .add_directive("patchwork_rust_core=trace".parse().unwrap()) + .add_directive("samod=info".parse().unwrap()) + .add_directive("samod_core=info".parse().unwrap()), + ); + + #[cfg(feature = "tokio-console")] + let subscriber = tracing_subscriber::registry() + .with( + console_subscriber::ConsoleLayer::builder() + .with_default_env() + .spawn(), + ) + .with(stdout_layer) + .with(file_layer) + .try_init(); + + #[cfg(not(feature = "tokio-console"))] + let subscriber = tracing_subscriber::registry() .with(stdout_layer) - // we want a file writer too .with(file_layer) - .try_init() - { + .try_init(); + + if let Err(e) = subscriber { tracing::error!("Failed to initialize tracing subscriber: {:?}", e); } else { tracing::info!("Tracing subscriber initialized"); } } - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct TimeNoDate { year: i64, @@ -94,7 +106,7 @@ impl fmt::Display for TimeNoDate { write!( f, // "-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", - "{:02}:{:02}:{:02}.{:06}", + "{:02}:{:02}:{:02}.{:06}", self.hour, self.minute, self.second, @@ -191,25 +203,25 @@ impl From for TimeNoDate { } // custom stdout Writer -pub struct CustomStdoutWriter{ - inner: std::io::Stdout, +pub struct CustomStdoutWriter { + inner: std::io::Stdout, } impl CustomStdoutWriter { - pub fn custom_stdout() -> CustomStdoutWriter { - CustomStdoutWriter { - inner: std::io::stdout(), - } - } + pub fn custom_stdout() -> CustomStdoutWriter { + CustomStdoutWriter { + inner: std::io::stdout(), + } + } } // the formatting in tracing-subscriber REALLY SUCKS, so we need to just search-and-replace the output strings // Search and replace for the level names const LEVEL_NAMES_TO_REPLACEMENT: &[(&str, &str)] = &[ - ("TRACE", "T"), - ("DEBUG", "D"), - (" INFO", "I"), // extra space because INFO is 4 letters long - (" WARN", "W"), - ("ERROR", "X"), + ("TRACE", "T"), + ("DEBUG", "D"), + (" INFO", "I"), // extra space because INFO is 4 letters long + (" WARN", "W"), + ("ERROR", "X"), ]; const CRATE_NAME: &str = "patchwork_rust_core"; @@ -222,9 +234,9 @@ impl Write for CustomStdoutWriter { LEVEL_NAMES_TO_REPLACEMENT.iter().fold(s.to_string(), |acc, (from, to)| acc.replace(from, to)) .replace(CRATE_NAME, ""); - let size_diff = buf.len() - s.len(); + let size_diff = buf.len() - s.len(); let actual_written = self.inner.write(s.as_bytes())?; - Ok(actual_written + size_diff) + Ok(actual_written + size_diff) } fn flush(&mut self) -> std::io::Result<()> { self.inner.flush() @@ -242,7 +254,6 @@ impl Write for CustomStdoutWriter { // } // } - // impl Write for CustomJSONStdoutWriter { // fn write(&mut self, buf: &[u8]) -> std::io::Result { // // deserialize the entire fucking json object @@ -265,8 +276,6 @@ impl Write for CustomStdoutWriter { // } // } - - // // This needs to be a seperate impl block because they place different bounds on the type parameters. // impl Layer // where @@ -342,7 +351,6 @@ impl Write for CustomStdoutWriter { // } // } - // impl FormatEvent // for fn(ctx: &FmtContext<'_, S, N>, Writer<'_>, &Event<'_>) -> fmt::Result // where @@ -372,7 +380,6 @@ impl Write for CustomStdoutWriter { // ) -> fmt::Result; // } - // impl FormatEventEXT for Format // where // S: Subscriber + for<'a> LookupSpan<'a>, diff --git a/rust/src/helpers/utils.rs b/rust/src/helpers/utils.rs index 556c1989..41289eaf 100644 --- a/rust/src/helpers/utils.rs +++ b/rust/src/helpers/utils.rs @@ -289,25 +289,3 @@ pub fn exact_human_readable_timestamp(timestamp: i64) -> String { let datetime : DateTime = DateTime::from(dt.unwrap()); return datetime.format("%Y-%m-%d %H:%M:%S").to_string(); } - -pub fn spawn_named(name: &str, future: F) -> JoinHandle where - F: Future + Send + 'static, - F::Output: Send + 'static, -{ - let span = tracing::info_span!("task", name = name); - tokio::task::Builder::new() - .name(name) - .spawn(future.instrument(span)) - .expect(&format!("Something went wrong trying to build the task {name}.")) -} - -pub fn spawn_named_on(name: &str, runtime: &runtime::Handle, future: F) -> JoinHandle where - F: Future + Send + 'static, - F::Output: Send + 'static, -{ - let span = tracing::info_span!("task", name = name); - tokio::task::Builder::new() - .name(name) - .spawn_on(future.instrument(span), runtime) - .expect(&format!("Something went wrong trying to build the task {name}.")) -} \ No newline at end of file diff --git a/rust/src/project/change_ingester.rs b/rust/src/project/change_ingester.rs index f4aa16ae..a5748840 100644 --- a/rust/src/project/change_ingester.rs +++ b/rust/src/project/change_ingester.rs @@ -11,7 +11,7 @@ use tokio::{ use tokio_util::sync::CancellationToken; use crate::{ - helpers::utils::{CommitInfo, CommitMetadata, spawn_named, summarize_changes}, + helpers::{spawn_utils::spawn_named, utils::{CommitInfo, CommitMetadata, summarize_changes}}, project::{branch_db::BranchDb, peer_watcher::PeerWatcher, project_api::ChangeViewModel}, }; diff --git a/rust/src/project/connection.rs b/rust/src/project/connection.rs index ba7e02f1..c52520a4 100644 --- a/rust/src/project/connection.rs +++ b/rust/src/project/connection.rs @@ -8,7 +8,7 @@ use tokio::{ use tokio_stream::wrappers::BroadcastStream; use tokio_util::sync::CancellationToken; -use crate::helpers::utils::spawn_named; +use crate::helpers::spawn_utils::spawn_named; #[derive(Debug, Clone)] enum ConnectionStoppedReason { diff --git a/rust/src/project/document_watcher.rs b/rust/src/project/document_watcher.rs index 6eb1308c..9e750415 100644 --- a/rust/src/project/document_watcher.rs +++ b/rust/src/project/document_watcher.rs @@ -9,9 +9,7 @@ use crate::{ branch::{ BranchState, BranchStateForkInfo, BranchStateMergeInfo, BranchStateRevertInfo, BranchesMetadataDoc, - }, - doc_utils::SimpleDocReader, - utils::{parse_automerge_url, spawn_named}, + }, doc_utils::SimpleDocReader, spawn_utils::spawn_named, utils::parse_automerge_url }, project::branch_db::BranchDb, }; diff --git a/rust/src/project/driver.rs b/rust/src/project/driver.rs index e3db3131..9b65d7eb 100644 --- a/rust/src/project/driver.rs +++ b/rust/src/project/driver.rs @@ -1,7 +1,8 @@ use crate::diff::differ::ProjectDiff; use crate::fs::file_utils::FileSystemEvent; use crate::helpers::branch::{self, BranchState}; -use crate::helpers::utils::{CommitInfo, CommitMetadata, spawn_named}; +use crate::helpers::spawn_utils::spawn_named; +use crate::helpers::utils::{CommitInfo, CommitMetadata}; use crate::interop::godot_accessors::{EditorFilesystemAccessor, PatchworkEditorAccessor}; use crate::project::branch_db::{BranchDb, HistoryRef}; use crate::project::change_ingester::ChangeIngester; diff --git a/rust/src/project/peer_watcher.rs b/rust/src/project/peer_watcher.rs index 9c77fe8d..7668203d 100644 --- a/rust/src/project/peer_watcher.rs +++ b/rust/src/project/peer_watcher.rs @@ -4,7 +4,7 @@ use tokio::{select, sync::watch}; use tokio_stream::wrappers::WatchStream; use tokio_util::sync::CancellationToken; -use crate::helpers::utils::spawn_named; +use crate::helpers::spawn_utils::spawn_named; #[derive(Debug)] pub struct PeerWatcher { diff --git a/rust/src/project/project.rs b/rust/src/project/project.rs index e8512f1e..7fe0551b 100644 --- a/rust/src/project/project.rs +++ b/rust/src/project/project.rs @@ -1,7 +1,8 @@ use crate::diff::differ::ProjectDiff; use crate::fs::file_utils::FileSystemEvent; use crate::helpers::branch::BranchState; -use crate::helpers::utils::{CommitInfo, spawn_named, spawn_named_on, summarize_changes}; +use crate::helpers::spawn_utils::spawn_named_on; +use crate::helpers::utils::{CommitInfo, summarize_changes}; use crate::interop::godot_accessors::{ EditorFilesystemAccessor, PatchworkConfigAccessor, PatchworkEditorAccessor, }; diff --git a/rust/src/project/project_api_impl.rs b/rust/src/project/project_api_impl.rs index 4a081492..8e9774ac 100644 --- a/rust/src/project/project_api_impl.rs +++ b/rust/src/project/project_api_impl.rs @@ -8,7 +8,7 @@ use crate::{ diff::differ::ProjectDiff, helpers::utils::{ BranchWrapper, CommitInfo, DiffWrapper, exact_human_readable_timestamp, - human_readable_timestamp, spawn_named, spawn_named_on, + human_readable_timestamp, }, interop::godot_accessors::PatchworkConfigAccessor, project::{ diff --git a/rust/src/project/sync_fs_to_automerge.rs b/rust/src/project/sync_fs_to_automerge.rs index b430e5ba..fc5b3536 100644 --- a/rust/src/project/sync_fs_to_automerge.rs +++ b/rust/src/project/sync_fs_to_automerge.rs @@ -6,7 +6,7 @@ use tokio_util::sync::CancellationToken; use tracing::instrument; use crate::{ - fs::file_utils::{FileContent, FileSystemEvent}, helpers::utils::spawn_named, project::{branch_db::BranchDb, fs_watcher::FileSystemWatcher} + fs::file_utils::{FileContent, FileSystemEvent}, helpers::spawn_utils::spawn_named, project::{branch_db::BranchDb, fs_watcher::FileSystemWatcher} }; /// Tracks changes using [FileSystemWatcher], handles the changes, and tracks them as pending. From 2af98a38a625b9487d0f64b2efade4a79add1cec Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Wed, 28 Jan 2026 06:39:45 -0800 Subject: [PATCH 12/49] misc fixes --- rust/src/diff/differ.rs | 6 ++---- rust/src/project/branch_db/commit.rs | 2 +- rust/src/project/branch_db/file.rs | 23 ++++++++++++----------- rust/src/project/branch_db/util.rs | 7 ++++++- rust/src/project/document_watcher.rs | 2 +- rust/src/project/driver.rs | 18 +++++++++++++++--- rust/src/project/fs_watcher.rs | 2 +- rust/src/project/project.rs | 3 ++- rust/src/project/sync_fs_to_automerge.rs | 3 ++- 9 files changed, 42 insertions(+), 24 deletions(-) diff --git a/rust/src/diff/differ.rs b/rust/src/diff/differ.rs index 8843fccb..aa3a780c 100644 --- a/rust/src/diff/differ.rs +++ b/rust/src/diff/differ.rs @@ -105,7 +105,7 @@ impl Differ { let temp_dir = format!("res://.patchwork/temp_{}/", temp_id); let temp_path = path.replace("res://", &temp_dir); if let Err(e) = content - .write(&PathBuf::from(self.branch_db.globalize_path(&temp_path))) + .write(&self.branch_db.globalize_path(&temp_path)) .await { tracing::error!("error writing file to temp path: {:?}", e); @@ -120,9 +120,7 @@ impl Differ { import_file_content.replace(r#"uid=uid://[^\n]+"#, "uid=uid://"); // write the import file content to the temp path let import_file_path: String = format!("{}.import", temp_path); - let _ = FileContent::String(import_file_content).write(&PathBuf::from( - self.branch_db.globalize_path(&import_file_path), - )); + let _ = FileContent::String(import_file_content).write(&self.branch_db.globalize_path(&import_file_path)); let res = PatchworkEditorAccessor::import_and_load_resource(&temp_path); if res.is_nil() { diff --git a/rust/src/project/branch_db/commit.rs b/rust/src/project/branch_db/commit.rs index f905d8b8..315e5a68 100644 --- a/rust/src/project/branch_db/commit.rs +++ b/rust/src/project/branch_db/commit.rs @@ -41,7 +41,7 @@ impl BranchDb { let count = files.len(); let username = self.username.lock().await.clone(); - if (count == 0) { + if count == 0 { tracing::info!("No actual changes found; not committing."); return None; } diff --git a/rust/src/project/branch_db/file.rs b/rust/src/project/branch_db/file.rs index c41ae72f..57e88cfc 100644 --- a/rust/src/project/branch_db/file.rs +++ b/rust/src/project/branch_db/file.rs @@ -77,14 +77,15 @@ impl BranchDb { if old_ref.is_none() || !old_ref.unwrap().is_valid() { tracing::info!("old heads empty, getting ALL files on branch"); + // NOTE: This returns local res:// paths, we must globalize them before exporting them to FileSystemEvents let files = self.get_files_at_ref(&new_ref, &HashSet::new()).await?; return Some( files .into_iter() .map(|(path, content)| match content { - FileContent::Deleted => FileSystemEvent::FileDeleted(PathBuf::from(path)), - _ => FileSystemEvent::FileCreated(PathBuf::from(path), content), + FileContent::Deleted => FileSystemEvent::FileDeleted(self.globalize_path(&path)), + _ => FileSystemEvent::FileCreated(self.globalize_path(&path), content), }) .collect(), ); @@ -103,21 +104,21 @@ impl BranchDb { let mut events = Vec::new(); for (path, _) in old_files.iter() { if !new_files.contains_key(path) { - events.push(FileSystemEvent::FileDeleted(PathBuf::from(path))); + events.push(FileSystemEvent::FileDeleted(self.globalize_path(path))); } } for (path, content) in new_files { match content { FileContent::Deleted => { - events.push(FileSystemEvent::FileDeleted(PathBuf::from(path))); + events.push(FileSystemEvent::FileDeleted(self.globalize_path(&path))); continue; } _ => {} } if !old_files.contains_key(&path) { - events.push(FileSystemEvent::FileCreated(PathBuf::from(path), content)); + events.push(FileSystemEvent::FileCreated(self.globalize_path(&path), content)); } else if &content != old_files.get(&path).unwrap() { - events.push(FileSystemEvent::FileModified(PathBuf::from(path), content)); + events.push(FileSystemEvent::FileModified(self.globalize_path(&path), content)); } } return Some(events); @@ -175,19 +176,19 @@ impl BranchDb { .await? .into_iter() .map(|(path, content)| match content { - FileContent::Deleted => FileSystemEvent::FileDeleted(PathBuf::from(path)), + FileContent::Deleted => FileSystemEvent::FileDeleted(self.globalize_path(&path)), _ if added_files.contains(&path) => { - FileSystemEvent::FileCreated(PathBuf::from(path), content) + FileSystemEvent::FileCreated(self.globalize_path(&path), content) } _ if deleted_files.contains(&path) => { - FileSystemEvent::FileDeleted(PathBuf::from(path)) + FileSystemEvent::FileDeleted(self.globalize_path(&path)) } - _ => FileSystemEvent::FileModified(PathBuf::from(path), content), + _ => FileSystemEvent::FileModified(self.globalize_path(&path), content), }) .chain( deleted_files .iter() - .map(|path| FileSystemEvent::FileDeleted(PathBuf::from(path))), + .map(|path| FileSystemEvent::FileDeleted(self.globalize_path(&path))), ) .collect(), ) diff --git a/rust/src/project/branch_db/util.rs b/rust/src/project/branch_db/util.rs index 3a961d4b..a38a4f77 100644 --- a/rust/src/project/branch_db/util.rs +++ b/rust/src/project/branch_db/util.rs @@ -8,11 +8,14 @@ use crate::{helpers::branch::BranchState, project::branch_db::{BranchDb, History // Utility methods for working with [BranchDb]. impl BranchDb { /// Turns a filesytem path into a project-local res:// path. + /// Local paths are represented with a [String], while global paths are represented with a [PathBuf]. + /// This is because local paths are a URL, not a filesystem path. pub fn localize_path(&self, path: &PathBuf) -> String { let path = path.to_string_lossy().replace("\\", "/"); let project_dir = self.project_dir.to_string_lossy().replace("\\", "/"); if path.starts_with(&project_dir) { - // TODO: this isn't teeechnically a Path, it's a URL... PathBuf is probably the wrong choice + // TODO: this isn't teeechnically a Path, it's a URL... PathBuf is probably the wrong choice. + // That's why we turn it into a string when we export! let thing = PathBuf::from("res://".to_string()) .join(PathBuf::from(&path[project_dir.len()..].to_string())); thing.to_string_lossy().to_string() @@ -22,6 +25,8 @@ impl BranchDb { } /// Convert a project URL like res:// into a local filesystem path. + /// Local paths are represented with a [String], while global paths are represented with a [PathBuf]. + /// This is because local paths are a URL, not a filesystem path. pub fn globalize_path(&self, path: &String) -> PathBuf { // trim the project_dir from the front of the path if path.starts_with("res://") { diff --git a/rust/src/project/document_watcher.rs b/rust/src/project/document_watcher.rs index 9e750415..358bd0a5 100644 --- a/rust/src/project/document_watcher.rs +++ b/rust/src/project/document_watcher.rs @@ -191,7 +191,7 @@ impl DocumentWatcherInner { .await .unwrap(); - self.branch_db.set_linked_docs_for_branch(handle.document_id(), linked_docs.values().cloned().collect()); + self.branch_db.set_linked_docs_for_branch(handle.document_id(), linked_docs.values().cloned().collect()).await; } #[tracing::instrument(skip_all)] diff --git a/rust/src/project/driver.rs b/rust/src/project/driver.rs index 9b65d7eb..f48a323d 100644 --- a/rust/src/project/driver.rs +++ b/rust/src/project/driver.rs @@ -128,6 +128,7 @@ impl Driver { main_thread_block: MainThreadBlock, server_url: String, project_path: PathBuf, + username: String, storage_directory: PathBuf, metadata_id: Option, ) -> Option { @@ -140,8 +141,6 @@ impl Driver { .load() .await; - let project_path = PathBuf::from(project_path); - // Construct the ignore globs, and link in the gitignores let ignore_globs = vec![ "**/.DS_Store", @@ -174,6 +173,13 @@ impl Driver { // Start the connection let connection = RemoteConnection::new(repo.clone(), server_url); let branch_db = BranchDb::new(repo.clone(), project_path, ignore_globs); + branch_db + .set_username(if username.trim() == "" { + None + } else { + Some(username.trim().to_string()) + }) + .await; let peer_watcher = Arc::new(PeerWatcher::new(repo.clone())); let sync_automerge_to_fs = SyncAutomergeToFileSystem::new(branch_db.clone()); let sync_fs_to_automerge = SyncFileSystemToAutomerge::new(branch_db.clone()); @@ -538,7 +544,13 @@ impl DriverInner { let current_ref = self.branch_db.get_checked_out_ref_mut(); let current_ref = current_ref.read().await; if let Some(current_ref) = current_ref.clone() { - return Some(current_ref); + if let Some(ref_) = self + .branch_db + .get_latest_ref_on_branch(¤t_ref.branch) + .await + { + return Some(ref_); + } } if let Some(main_branch) = self.branch_db.get_main_branch().await { if let Some(ref_) = self.branch_db.get_latest_ref_on_branch(&main_branch).await { diff --git a/rust/src/project/fs_watcher.rs b/rust/src/project/fs_watcher.rs index 163dfaa7..e73f5772 100644 --- a/rust/src/project/fs_watcher.rs +++ b/rust/src/project/fs_watcher.rs @@ -224,7 +224,7 @@ impl FileSystemWatcher { }; tracing::debug!("Heard filesystem event list of {:?} items", notify_events.len()); for notify_event in notify_events { - tracing::debug!("Heard filesystem event {:?}", notify_event); + tracing::trace!("Heard filesystem event {:?}", notify_event); if let Some(evt) = this.process_notify_event(notify_event).await { yield evt; } diff --git a/rust/src/project/project.rs b/rust/src/project/project.rs index 7fe0551b..c5f91f0e 100644 --- a/rust/src/project/project.rs +++ b/rust/src/project/project.rs @@ -179,6 +179,7 @@ impl Project { ); let project_dir = self.project_dir.clone(); + let username = PatchworkConfigAccessor::get_user_value("user_name", ""); let block = self.main_thread_block.clone(); // TODO: Don't block on main thread for checkin @@ -187,7 +188,7 @@ impl Project { .block_on( // I think it's correct to spawn this on a different task explicitly, because block_on runs the future on the current thread, not a worker thread. spawn_named_on("Create driver", self.runtime.handle(), async move { - Driver::new(block, server_url, project_dir, storage_dir, metadata_id).await + Driver::new(block, server_url, project_dir, username, storage_dir, metadata_id).await }), ) .unwrap(); diff --git a/rust/src/project/sync_fs_to_automerge.rs b/rust/src/project/sync_fs_to_automerge.rs index fc5b3536..2933d021 100644 --- a/rust/src/project/sync_fs_to_automerge.rs +++ b/rust/src/project/sync_fs_to_automerge.rs @@ -116,7 +116,8 @@ impl SyncFileSystemToAutomerge { *checked_out_ref = Some(new_ref); return true; } else { - tracing::error!("Could not commit pending files! Making no changes."); + tracing::info!("Did not commit pending files!"); + pending_changes.clear(); return false; } } From 4d6012980ee42f800901ed9f364728406c1b887c Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 28 Jan 2026 14:45:46 -0800 Subject: [PATCH 13/49] fix bug in `parse_scene` where main resources did not have correct types assigned --- rust/src/parser/godot_parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/src/parser/godot_parser.rs b/rust/src/parser/godot_parser.rs index e91e4f2a..afd4146d 100644 --- a/rust/src/parser/godot_parser.rs +++ b/rust/src/parser/godot_parser.rs @@ -720,7 +720,7 @@ pub fn parse_scene(source: &String) -> Result { } else if section_id == "resource" { main_resource = Some(SubResourceNode { id: "".to_string(), // Resource sections don't have IDs - resource_type, + resource_type: scene_metadata.as_ref().map(|s| s.resource_type.clone()).unwrap_or("".to_string()), properties: properties.into_iter().collect(), idx: 0, }); From d93f86cb3569b4da7bf2c4cafab17a9acbcd9d20 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 28 Jan 2026 09:11:03 -0800 Subject: [PATCH 14/49] Add text resource diffing (rebase) --- rust/src/diff/differ.rs | 33 +++- rust/src/diff/resource_differ.rs | 15 +- rust/src/diff/scene_differ.rs | 258 ++++++++++++++++++++++++++++--- rust/src/interop/godot_diffs.rs | 63 +++++++- 4 files changed, 327 insertions(+), 42 deletions(-) diff --git a/rust/src/diff/differ.rs b/rust/src/diff/differ.rs index aa3a780c..a090a694 100644 --- a/rust/src/diff/differ.rs +++ b/rust/src/diff/differ.rs @@ -14,13 +14,14 @@ use godot::{ use tracing::instrument; use crate::{ - diff::{resource_differ::ResourceDiff, scene_differ::SceneDiff, text_differ::TextDiff}, + diff::{resource_differ::BinaryResourceDiff, scene_differ::{SceneDiff, TextResourceDiff}, text_differ::TextDiff}, fs::{file_utils::FileSystemEvent, file_utils::FileContent}, helpers::{branch::BranchState, utils::ToShortForm}, interop::godot_accessors::PatchworkEditorAccessor, project::{ branch_db::{BranchDb, HistoryRef}, }, + project::project::Project, }; /// The type of change that occurred in a diff. @@ -39,8 +40,10 @@ pub enum ChangeType { pub enum Diff { /// A scene file diff. Scene(SceneDiff), + /// A text resource diff. + TextResourceDiff(TextResourceDiff), /// A resource file diff. - Resource(ResourceDiff), + BinaryResource(BinaryResourceDiff), /// A text file diff. Text(TextDiff), } @@ -266,17 +269,31 @@ impl Differ { FileContent::Scene(s) => Some(s), _ => None, }; - // This is a scene file, so use a scene diff - diffs.push(Diff::Scene( - self.get_scene_diff(&path, old_scene, new_scene, before, after) + + + let resource_type = match (old_scene, new_scene) { + (None, Some(scene)) => scene.resource_type.clone(), + (Some(scene), None) => scene.resource_type.clone(), + (_, Some(scene)) => scene.resource_type.clone(), + (_, _) => "".to_string(), + }; + if resource_type == "PackedScene" { + diffs.push(Diff::Scene( + self.get_scene_diff(&path, old_scene, new_scene, before, after) .await, - )); + )); + } else { + diffs.push(Diff::TextResourceDiff( + self.get_text_resource_diff(&path, old_scene, new_scene, before, after) + .await, + )); + } } else if matches!(old_file_content, FileContent::Binary(_)) || matches!(new_file_content, FileContent::Binary(_)) { // This is a binary file, so use a resource diff - diffs.push(Diff::Resource( - self.get_resource_diff( + diffs.push(Diff::BinaryResource( + self.get_binary_resource_diff( &path, change_type, old_file_content, diff --git a/rust/src/diff/resource_differ.rs b/rust/src/diff/resource_differ.rs index e540d19a..caf7079e 100644 --- a/rust/src/diff/resource_differ.rs +++ b/rust/src/diff/resource_differ.rs @@ -4,22 +4,23 @@ use crate::{ diff::differ::{ChangeType, Differ}, fs::file_utils::FileContent, helpers::utils::ToShortForm, project::branch_db::HistoryRef }; + #[derive(Clone, Debug)] -pub struct ResourceDiff { +pub struct BinaryResourceDiff { pub path: String, pub change_type: ChangeType, pub old_resource: Option, pub new_resource: Option, } -impl ResourceDiff { +impl BinaryResourceDiff { pub fn new( path: String, change_type: ChangeType, old_resource: Option, new_resource: Option, - ) -> ResourceDiff { - ResourceDiff { + ) -> BinaryResourceDiff { + BinaryResourceDiff { path, change_type, old_resource, @@ -29,7 +30,7 @@ impl ResourceDiff { } impl Differ { - pub(super) async fn get_resource_diff( + pub(super) async fn get_binary_resource_diff( &self, path: &String, change_type: ChangeType, @@ -37,8 +38,8 @@ impl Differ { new_content: &FileContent, before: &HistoryRef, after: &HistoryRef - ) -> ResourceDiff { - ResourceDiff::new( + ) -> BinaryResourceDiff { + BinaryResourceDiff::new( path.clone(), change_type, self.get_resource(path, old_content, before).await, diff --git a/rust/src/diff/scene_differ.rs b/rust/src/diff/scene_differ.rs index 9518f767..757eb7bd 100644 --- a/rust/src/diff/scene_differ.rs +++ b/rust/src/diff/scene_differ.rs @@ -11,7 +11,7 @@ use godot::{ use crate::{ diff::differ::{ChangeType, Differ}, parser::godot_parser::{ - ExternalResourceNode, GodotNode, GodotScene, SubResourceNode, TypeOrInstance, + ExternalResourceNode, GodotNode, GodotScene, OrderedProperty, SubResourceNode, TypeOrInstance }, project::branch_db::HistoryRef, }; @@ -37,6 +37,31 @@ impl SceneDiff { } } +#[derive(Clone, Debug)] +pub struct TextResourceDiff { + /// The path of the scene. + pub path: String, + /// PackedScene or other + pub resource_type: String, + /// The change type for the scene. + pub change_type: ChangeType, + /// The sub resources changed in this diff. + pub changed_sub_resources: Vec, + pub changed_main_resource: Option, +} + +impl TextResourceDiff { + fn new(path: String, resource_type: String, change_type: ChangeType, changed_sub_resources: Vec, changed_main_resource: Option) -> TextResourceDiff { + TextResourceDiff { + path, + resource_type, + change_type, + changed_sub_resources, + changed_main_resource, + } + } +} + /// Represents a diff of a single node within a scene, with a collection of changed properties. #[derive(Clone, Debug)] pub struct NodeDiff { @@ -66,6 +91,31 @@ impl NodeDiff { } } + + +#[derive(Clone, Debug)] +pub struct SubResourceDiff { + pub change_type: ChangeType, + pub sub_resource_id: String, + pub resource_type: String, + pub changed_properties: HashMap, +} + +impl SubResourceDiff { + pub fn new( + change_type: ChangeType, + sub_resource_id: String, + resource_type: String, + changed_properties: HashMap, + ) -> SubResourceDiff { + SubResourceDiff { + change_type, + sub_resource_id, + resource_type, + changed_properties, + } + } +} /// Represents a diff of a single Property within a Node, within a Scene. #[derive(Clone, Debug)] pub struct PropertyDiff { @@ -120,6 +170,50 @@ impl std::fmt::Display for VariantStrValue { } } +trait PropertyGetter { + fn get_property(&self, prop: &String) -> Option<&OrderedProperty>; + fn get_properties(&self) -> &HashMap; + fn get_type_or_instance(&self) -> TypeOrInstance; + fn is_subresource(&self) -> bool; + fn get_id(&self) -> String; +} + +impl PropertyGetter for GodotNode { + fn get_property(&self, prop: &String) -> Option<&OrderedProperty> { + self.properties.get(prop) + } + fn get_properties(&self) -> &HashMap { + &self.properties + } + fn get_type_or_instance(&self) -> TypeOrInstance { + self.type_or_instance.clone() + } + fn is_subresource(&self) -> bool { + false + } + fn get_id(&self) -> String { + self.id.to_string() + } +} + +impl PropertyGetter for SubResourceNode { + fn get_property(&self, prop: &String) -> Option<&OrderedProperty> { + self.properties.get(prop) + } + fn get_properties(&self) -> &HashMap { + &self.properties + } + fn get_type_or_instance(&self) -> TypeOrInstance { + TypeOrInstance::Type(self.resource_type.clone()) + } + fn is_subresource(&self) -> bool { + true + } + fn get_id(&self) -> String { + self.id.to_string() + } +} + /// Implement scene-related functions on the Differ impl Differ { /// Generate a [SceneDiff] between the previous and current heads. @@ -180,12 +274,124 @@ impl Differ { ) } + pub(super) async fn get_text_resource_diff( + &self, + path: &String, + old_scene: Option<&GodotScene>, + new_scene: Option<&GodotScene>, + before: &HistoryRef, + after: &HistoryRef, + ) -> TextResourceDiff { + let mut node_ids = HashSet::new(); + let mut sub_resource_ids = HashSet::new(); + let mut ext_resource_ids = HashSet::new(); + + let mut resource_type: String = "".to_string(); + // Collect all the relevant node IDs, sub resource IDs, and ext resource IDs from both scenes. + if let Some(old_scene) = old_scene { + resource_type = old_scene.resource_type.clone(); + Self::get_ids_from_scene( + old_scene, + &mut node_ids, + &mut ext_resource_ids, + &mut sub_resource_ids, + ); + } + if let Some(new_scene) = new_scene { + resource_type = new_scene.resource_type.clone(); + Self::get_ids_from_scene( + new_scene, + &mut node_ids, + &mut ext_resource_ids, + &mut sub_resource_ids, + ); + } + + let mut changed_sub_resources = Vec::new(); + // Diff each node + for sub_resource_id in &sub_resource_ids { + let old_sub_resource = old_scene.as_ref().and_then(|s| s.sub_resources.get(sub_resource_id)); + let new_sub_resource = new_scene.as_ref().and_then(|s| s.sub_resources.get(sub_resource_id)); + + let Some(diff) = self.get_sub_resource_diff(sub_resource_id, old_sub_resource, new_sub_resource, old_scene, new_scene, before, after).await + else { + // If the node has no changes or is otherwise invalid, just skip this one. + continue; + }; + + changed_sub_resources.push(diff); + } + let changed_main_resource = self.get_sub_resource_diff(&"".to_string(), old_scene.and_then(|s| s.main_resource.as_ref()), new_scene.and_then(|s| s.main_resource.as_ref()), old_scene, new_scene, before, after).await; + + TextResourceDiff::new( + path.clone(), + resource_type, + match (old_scene, new_scene) { + (None, Some(_)) => ChangeType::Added, + (Some(_), None) => ChangeType::Removed, + (_, _) => ChangeType::Modified, + }, + changed_sub_resources, + changed_main_resource, + ) + + } + + async fn get_sub_resource_diff( + &self, + sub_resource_id: &String, + old_node: Option<&SubResourceNode>, + new_node: Option<&SubResourceNode>, + old_scene: Option<&GodotScene>, + new_scene: Option<&GodotScene>, + before: &HistoryRef, + after: &HistoryRef, + ) -> Option { + if old_node.is_none() && new_node.is_none() { + return None; + } + + let mut changed_properties = HashMap::new(); + let old_class_name = old_node.map(|n| n.get_type_or_instance().to_string()); + let new_class_name = new_node.map(|n| n.get_type_or_instance().to_string()); + + // Collect all properties from new and old scenes + let mut props: HashSet = HashSet::new(); + if let Some(node) = old_node { + for (key, _) in node.get_properties() { + let _ = props.insert(key.to_string()); + } + } + if let Some(node) = new_node { + for (key, _) in node.get_properties() { + let _ = props.insert(key.to_string()); + } + } + for prop in &props { + if let Some(prop_diff) = + self.get_property_diff(prop, old_node, new_node, old_scene, new_scene, before, after).await + { + changed_properties.insert(prop.clone(), prop_diff); + } + } + Some(SubResourceDiff::new( + match (old_node, new_node) { + (None, Some(_)) => ChangeType::Added, + (Some(_), None) => ChangeType::Removed, + (_, _) => ChangeType::Modified, + }, + sub_resource_id.clone(), + old_class_name.or(new_class_name)?, + changed_properties, + )) + } + /// Generate a [NodeDiff] between two nodes. async fn get_node_diff( &self, node_id: i32, - old_node: Option<&GodotNode>, - new_node: Option<&GodotNode>, + old_node: Option<&impl PropertyGetter>, + new_node: Option<&impl PropertyGetter>, old_scene: Option<&GodotScene>, new_scene: Option<&GodotScene>, before: &HistoryRef, @@ -195,20 +401,20 @@ impl Differ { return None; } - let old_class_name = old_node.map(|n| Self::get_class_name(&n.type_or_instance, old_scene)); - let new_class_name = new_node.map(|n| Self::get_class_name(&n.type_or_instance, new_scene)); + let old_class_name = old_node.map(|n| Self::get_class_name(&n.get_type_or_instance(), old_scene)); + let new_class_name = new_node.map(|n| Self::get_class_name(&n.get_type_or_instance(), new_scene)); let mut changed_properties = HashMap::new(); // Collect all properties from new and old scenes let mut props: HashSet = HashSet::new(); if let Some(node) = old_node { - for (key, _) in &node.properties { + for (key, _) in node.get_properties() { let _ = props.insert(key.to_string()); } } if let Some(node) = new_node { - for (key, _) in &node.properties { + for (key, _) in node.get_properties() { let _ = props.insert(key.to_string()); } } @@ -267,27 +473,37 @@ impl Differ { } } + fn get_classdb_default_value(class_name: &String, prop: &String) -> String { + if (ClassDb::singleton().is_instance_valid() && ClassDb::singleton().class_exists(class_name)) { + ClassDb::singleton() + .class_get_property_default_value( + &StringName::from(class_name), + &StringName::from(prop), + ) + .to_string() + } else { + "".to_string() + } + } + /// Returns the [VariantStrValue] of a property on a node, or the default value if the property doesn't /// exist on the node. /// If the node itself doesn't exist, returns [None]. - fn get_varstr_or_default(prop: &String, node: Option<&GodotNode>) -> Option { + fn get_varstr_or_default(prop: &String, node: Option<&impl PropertyGetter>) -> Option { // If this node never existed, don't provide a value. let Some(node) = node else { return None; }; - - let val = node.properties.get(prop).map_or_else( + let val = node.get_property(prop).map_or_else( || // If the property doesn't exist on the node, calculate the default. - match &node.type_or_instance { - TypeOrInstance::Type(class_name) => ClassDb::singleton() - .class_get_property_default_value( - &StringName::from(class_name), - &StringName::from(prop), - ) - .to_string(), - // Instance properties are always set, regardless of the default value, so this is always empty - _ => "".to_string(), + match &node.get_type_or_instance() { + TypeOrInstance::Type(class_name) => Self::get_classdb_default_value(class_name, prop), + TypeOrInstance::Instance(class_name) => match node.is_subresource() { + true => Self::get_classdb_default_value(class_name, prop), + // Instance properties are always set on Nodes, regardless of the default value, so this is going to be unused. + false => "".to_string(), + }, }, // Otherwise, get the value from the property. |val| val.get_value(), @@ -301,8 +517,8 @@ impl Differ { async fn get_property_diff( &self, prop: &String, - old_node: Option<&GodotNode>, - new_node: Option<&GodotNode>, + old_node: Option<&impl PropertyGetter>, + new_node: Option<&impl PropertyGetter>, old_scene: Option<&GodotScene>, new_scene: Option<&GodotScene>, before: &HistoryRef, diff --git a/rust/src/interop/godot_diffs.rs b/rust/src/interop/godot_diffs.rs index 14a73516..5c182d4b 100644 --- a/rust/src/interop/godot_diffs.rs +++ b/rust/src/interop/godot_diffs.rs @@ -9,8 +9,8 @@ use godot::obj::Singleton; use crate::{ diff::{ differ::{ChangeType, Diff, ProjectDiff}, - resource_differ::ResourceDiff, - scene_differ::{NodeDiff, PropertyDiff, SceneDiff}, + resource_differ::BinaryResourceDiff, + scene_differ::{NodeDiff, PropertyDiff, SceneDiff, SubResourceDiff, TextResourceDiff}, text_differ::{TextDiff, TextDiffHunk, TextDiffLine}, }, interop::godot_helpers::{GodotConvertExt, ToGodotExt}, @@ -104,7 +104,8 @@ impl ToGodot for Diff { fn to_godot(&self) -> ToArg<'_, Self::Via, Self::Pass> { match self { Diff::Scene(diff) => diff.to_godot(), - Diff::Resource(diff) => diff.to_godot(), + Diff::TextResourceDiff(diff) => diff.to_godot(), + Diff::BinaryResource(diff) => diff.to_godot(), Diff::Text(diff) => diff.to_godot(), } } @@ -122,7 +123,8 @@ impl ToGodot for ProjectDiff { dict.set( match diff { Diff::Scene(scene_diff) => scene_diff.path.clone(), - Diff::Resource(resource_diff) => resource_diff.path.clone(), + Diff::TextResourceDiff(scene_diff) => scene_diff.path.clone(), + Diff::BinaryResource(resource_diff) => resource_diff.path.clone(), Diff::Text(text_diff) => text_diff.path.clone(), } .to_godot(), @@ -148,6 +150,55 @@ impl ToGodot for SceneDiff { } } +impl GodotConvert for TextResourceDiff { + type Via = VarDictionary; +} + +impl ToGodot for TextResourceDiff { + type Pass = ByValue; + fn to_godot(&self) -> ToArg<'_, Self::Via, Self::Pass> { + vdict! { + "change_type": self.change_type.to_godot(), + "resource_type": self.resource_type.to_godot(), + "changed_sub_resources": self.changed_sub_resources.to_godot(), + "changed_main_resource": self.changed_main_resource.as_ref().map(|s| s.to_godot().to_variant()).unwrap_or(Variant::nil()), + "diff_type": "text_resource_changed", + } + } +} + +impl GodotConvert for SubResourceDiff { + type Via = VarDictionary; +} + +impl ToGodot for SubResourceDiff { + type Pass = ByValue; + fn to_godot(&self) -> ToArg<'_, Self::Via, Self::Pass> { + vdict! { + "change_type": self.change_type.to_godot(), + "sub_resource_id": self.sub_resource_id.to_godot(), + "resource_type": self.resource_type.to_godot(), + "changed_props": self.changed_properties.to_godot(), + } + } +} + +impl GodotConvertExt for Vec { + type Via = Array; +} + +impl ToGodotExt for Vec { + type Pass = ByValue; + fn _to_godot(&self) -> Array { + self.iter() + .map(|s| s.to_godot()) + .collect::>() + } + fn _to_variant(&self) -> Variant { + self._to_godot().to_variant() + } +} + impl GodotConvertExt for Vec { type Via = Array; } @@ -214,11 +265,11 @@ impl ToGodot for PropertyDiff { } } -impl GodotConvert for ResourceDiff { +impl GodotConvert for BinaryResourceDiff { type Via = VarDictionary; } -impl ToGodot for ResourceDiff { +impl ToGodot for BinaryResourceDiff { type Pass = ByValue; fn to_godot(&self) -> ToArg<'_, Self::Via, Self::Pass> { vdict! { From b2a2197f5ff135f028cfe6e8adb54f9cda59302c Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 28 Jan 2026 14:45:57 -0800 Subject: [PATCH 15/49] Working GUI text resource diff --- public/gdscript/diff_inspector_container.gd | 62 +++++++++++++++++---- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/public/gdscript/diff_inspector_container.gd b/public/gdscript/diff_inspector_container.gd index a34fc88d..db3e779d 100644 --- a/public/gdscript/diff_inspector_container.gd +++ b/public/gdscript/diff_inspector_container.gd @@ -87,16 +87,6 @@ func update_property_editor(editor_property) -> void: editor_property.update_property() editor_property._update_editor_property_status() -func getDeletedNodes() -> Array: - return deleted_nodes - -func getAddedNodes() -> Array: - return added_nodes - -func getChangedNodes() -> Array: - return changed_nodes - - func get_diff_stylebox(color: Color) -> StyleBoxTexture: var stylebox: StyleBoxTexture = StyleBoxTexture.new() stylebox.texture = diff_stylebox_tex @@ -541,6 +531,53 @@ func add_node_diff(file_section: DiffInspectorSection, file_path: String, node_d pop_node_sections(key, child_map[key], file_section, file_path) +func add_text_resource_diff(inspector_section: DiffInspectorSection, changed_sub_resources: Array, changed_main_resource: Dictionary) -> void: + inspector_section.get_vbox().add_child(HSeparator.new()) + if changed_main_resource is Dictionary and changed_main_resource.size() > 0: + add_sub_resource_diff(inspector_section, changed_main_resource["change_type"], changed_main_resource["sub_resource_id"], changed_main_resource["resource_type"], changed_main_resource["changed_props"]) + + for sub_resource in changed_sub_resources: + var change_type = sub_resource["change_type"] + var sub_resource_id = sub_resource["sub_resource_id"] + var sub_resource_type = sub_resource["resource_type"] + var changed_properties = sub_resource["changed_props"] + add_sub_resource_diff(inspector_section, change_type, sub_resource_id, sub_resource_type, changed_properties) + +func add_sub_resource_diff(inspector_section: DiffInspectorSection, change_type: String, sub_resource_id: String, sub_resource_type: String, changed_properties: Dictionary) -> void: + if (changed_properties.size() == 0 and change_type == "modified"): + print("!!! no prop diffs for ", sub_resource_id, " with type ", change_type) + return + var color: Color = modified_color + var subresource_label = sub_resource_id + if subresource_label.is_empty(): + subresource_label = "Main (" + sub_resource_type + ")" + if change_type == "added": + color = added_color + subresource_label += " (Added)" + elif change_type == "removed": + color = removed_color + subresource_label += " (Deleted)" + elif change_type == "modified": + color = modified_color + subresource_label += " (Modified)" + + var i = 0 + var child_section: DiffInspectorSection = DiffInspectorSection.new() + var fake_node: MissingResource = MissingResource.new() + fake_node.original_class = sub_resource_type + child_section.setup(sub_resource_id, subresource_label, fake_node, color, true, 1, 2) + child_section.set_type(change_type) + var vbox = child_section.get_vbox() + vbox.add_child(HSeparator.new()) + # get the length of the prop_diffs dictionary + for prop_name in changed_properties.keys(): + if i > 0: + var divider = HSeparator.new() + vbox.add_child(divider) + add_PropertyDiffResult(child_section, changed_properties[prop_name], sub_resource_type) + i += 1 + inspector_section.get_vbox().add_child(child_section) + func add_FileDiffResult(file_path: String, file_diff: Dictionary) -> void: @@ -580,6 +617,11 @@ func add_FileDiffResult(file_path: String, file_diff: Dictionary) -> void: var text_diff = file_diff["text_diff"] add_text_diff(inspector_section, text_diff) inspector_section.connect("box_clicked", self._on_text_box_clicked) + elif type == "text_resource_changed": + var changed_sub_resources = file_diff["changed_sub_resources"] + var changed_main_resource = file_diff["changed_main_resource"] + add_text_resource_diff(inspector_section, changed_sub_resources, changed_main_resource) + inspector_section.connect("box_clicked", self._on_resource_box_clicked) elif type == "scene_changed": var node_diffs: Array = file_diff["changed_nodes"] node_diffs.sort_custom(func(a, b): return a["node_path"] < b["node_path"]) From 14c93e883e733cd5c9aba3f09110cb8abb1a1ea4 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 28 Jan 2026 15:35:59 -0800 Subject: [PATCH 16/49] add GodotScene::get_ext_resource_path --- rust/src/parser/godot_parser.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/src/parser/godot_parser.rs b/rust/src/parser/godot_parser.rs index afd4146d..7bce2f0e 100644 --- a/rust/src/parser/godot_parser.rs +++ b/rust/src/parser/godot_parser.rs @@ -549,6 +549,10 @@ impl GodotScene { self.nodes.get(&node_id) } + pub fn get_ext_resource_path(&self, ext_resource_id: &String) -> Option { + self.ext_resources.get(ext_resource_id).map(|ext_resource| ext_resource.path.clone()) + } + } #[inline] From 81c35bbdb57458cfe12068d65c47d86787a4cb98 Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Thu, 29 Jan 2026 02:13:04 -0800 Subject: [PATCH 17/49] Fix for doc ID not saving to a file and changes not ingesting --- rust/src/project/change_ingester.rs | 2 +- rust/src/project/driver.rs | 1 + rust/src/project/project.rs | 27 ++++++++++++++------------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/rust/src/project/change_ingester.rs b/rust/src/project/change_ingester.rs index a5748840..52bbc12c 100644 --- a/rust/src/project/change_ingester.rs +++ b/rust/src/project/change_ingester.rs @@ -106,7 +106,7 @@ impl ChangeIngesterInner { // since we're past the duration with no other requests, the counter resets. *last_ingest = (now, 0); } - self.changes_tx.send(self.get_changes().await).unwrap(); + self.changes_tx.send_replace(self.get_changes().await); last_ingest.1 += 1; } diff --git a/rust/src/project/driver.rs b/rust/src/project/driver.rs index f48a323d..16b4e7ec 100644 --- a/rust/src/project/driver.rs +++ b/rust/src/project/driver.rs @@ -517,6 +517,7 @@ impl DriverInner { if new_checked_out_ref.as_ref().map(|r| &r.branch) != old_checked_out_ref.as_ref().map(|r| &r.branch) { + self.change_ingester.request_ingestion(); self.ref_tx.send(new_checked_out_ref).unwrap(); } tracing::trace!("Done with sync."); diff --git a/rust/src/project/project.rs b/rust/src/project/project.rs index c5f91f0e..ae4f3727 100644 --- a/rust/src/project/project.rs +++ b/rust/src/project/project.rs @@ -170,9 +170,7 @@ impl Project { }, None => None, }; - - // TODO (Lilith): Add support back for initially checking out a branch; probably once we're syncing - // let checked_out_branch_doc_id = PatchworkConfigAccessor::get_project_value("checked_out_branch_doc_id", ""); + tracing::info!( "Starting GodotProject with metadata doc id: {:?}", metadata_id @@ -183,24 +181,28 @@ impl Project { let block = self.main_thread_block.clone(); // TODO: Don't block on main thread for checkin - *self.driver.blocking_lock() = self + let (driver, metadata) = self .runtime .block_on( // I think it's correct to spawn this on a different task explicitly, because block_on runs the future on the current thread, not a worker thread. spawn_named_on("Create driver", self.runtime.handle(), async move { - Driver::new(block, server_url, project_dir, username, storage_dir, metadata_id).await + let driver = Driver::new(block, server_url, project_dir, username, storage_dir, metadata_id).await; + let metadata = driver.as_ref().unwrap().get_metadata_doc().await; + (driver, metadata) }), ) .unwrap(); - if self.driver.blocking_lock().is_none() { + let Some(driver) = driver else { tracing::error!("Could not start the driver!"); return; - } + }; - let driver = self.driver.blocking_lock(); - self.changes_rx = Some(driver.as_ref().unwrap().get_changes_rx()); - self.checked_out_ref_rx = Some(driver.as_ref().unwrap().get_ref_rx()); + PatchworkConfigAccessor::set_project_value("project_doc_id", &metadata.expect("Driver started, but metadata null!").to_string()); + self.changes_rx = Some(driver.get_changes_rx()); + self.checked_out_ref_rx = Some(driver.get_ref_rx()); + + *self.driver.blocking_lock() = Some(driver); } pub fn stop(&mut self) { @@ -289,13 +291,12 @@ impl Project { // Check to see if we need to produce a CheckedOutBranch signal let rx = self.checked_out_ref_rx.as_mut().unwrap(); if rx.has_changed().unwrap_or(false) { + let doc_id = rx.borrow().clone().map(|r| r.branch.to_string()).unwrap_or("".to_string()); + PatchworkConfigAccessor::set_project_value("checked_out_branch_doc_id", &doc_id); signals.push(GodotProjectSignal::CheckedOutBranch); rx.mark_unchanged(); } - // TODO (Lilith): VERY IMPORTANT, set the patchwork config branch ID here!!! - // So that we save the branch ID for future checkouts. - tracing::trace!("Done with process."); (fs_changes, signals) } From c5c266b97249024bca8ecb3cc213bb1438266ab7 Mon Sep 17 00:00:00 2001 From: Lilith Silver <84940819+LilithSilver@users.noreply.github.com> Date: Thu, 29 Jan 2026 02:29:02 -0800 Subject: [PATCH 18/49] Fix branch picker --- rust/src/project/project_api_impl.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/src/project/project_api_impl.rs b/rust/src/project/project_api_impl.rs index 8e9774ac..fe7597c3 100644 --- a/rust/src/project/project_api_impl.rs +++ b/rust/src/project/project_api_impl.rs @@ -334,8 +334,7 @@ impl ProjectViewModel for Project { Some(BranchWrapper { state: state.clone(), - // children, - children: Vec::new(), + children, }) } From f1c403bce8be18b2c2c20dd20597131dee27c658 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Thu, 29 Jan 2026 15:07:24 -0800 Subject: [PATCH 19/49] add PatchworkEditor::import_and_save_resource_to_temp --- editor/patchwork_editor.cpp | 16 ++++++++++++---- editor/patchwork_editor.h | 2 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/editor/patchwork_editor.cpp b/editor/patchwork_editor.cpp index 2c50a6b9..0cfb03fc 100644 --- a/editor/patchwork_editor.cpp +++ b/editor/patchwork_editor.cpp @@ -412,7 +412,7 @@ inline Vector _get_section_keys(const Ref &p_config_file, co #endif } -Ref PatchworkEditor::import_and_load_resource(const String &p_path) { +String PatchworkEditor::import_and_save_resource_to_temp(const String &p_path) { // get the import path auto import_path = p_path + ".import"; // load the import file @@ -463,10 +463,17 @@ Ref PatchworkEditor::import_and_load_resource(const String &p_path) { err = importer->import(ResourceUID::INVALID_ID, p_path, import_base_path, params, &import_variants, &import_options, &metadata); ERR_FAIL_COND_V_MSG(err != OK, {}, "Failed to import resource at path " + p_path); + return import_base_path; + +} + +Ref PatchworkEditor::import_and_load_resource(const String &p_path) { + auto import_base_path = import_and_save_resource_to_temp(p_path); // load the resource - auto res = ResourceLoader::load(p_path, "", ResourceFormatLoader::CACHE_MODE_IGNORE_DEEP, &err); - ERR_FAIL_COND_V_MSG(err != OK, {}, "Failed to load resource at path " + import_base_path); - return res; + if (import_base_path.is_empty()) { + return nullptr; + } + return ResourceLoader::load(import_base_path, "", ResourceFormatLoader::CACHE_MODE_IGNORE_DEEP); } void PatchworkEditor::save_all_scenes_and_scripts() { @@ -693,6 +700,7 @@ void PatchworkEditor::_bind_methods() { ClassDB::bind_static_method(get_class_static(), D_METHOD("get_recursive_dir_list", "dir", "wildcards", "absolute", "rel"), &PatchworkEditor::get_recursive_dir_list); ClassDB::bind_static_method(get_class_static(), D_METHOD("get_importer_by_name", "name"), &PatchworkEditor::get_importer_by_name); + ClassDB::bind_static_method(get_class_static(), D_METHOD("import_and_save_resource_to_temp", "path"), &PatchworkEditor::import_and_save_resource_to_temp); ClassDB::bind_static_method(get_class_static(), D_METHOD("import_and_load_resource", "path"), &PatchworkEditor::import_and_load_resource); ClassDB::bind_static_method(get_class_static(), D_METHOD("save_all_scenes_and_scripts"), &PatchworkEditor::save_all_scenes_and_scripts); diff --git a/editor/patchwork_editor.h b/editor/patchwork_editor.h index 6e126f94..57931b8f 100644 --- a/editor/patchwork_editor.h +++ b/editor/patchwork_editor.h @@ -43,6 +43,8 @@ class PatchworkEditor : public Node { static void progress_task_step_bg(const String &p_task, int p_step = -1); static void progress_end_task_bg(const String &p_task); static Ref get_importer_by_name(const String &p_name); + static String import_and_save_resource_to_temp(const String &p_path); + static Ref import_and_load_resource(const String &p_path); static Vector get_unsaved_files(); From aac6c8fa2394068bf0d7df72a2d2a1240871d9cc Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Thu, 29 Jan 2026 15:32:14 -0800 Subject: [PATCH 20/49] add lazy load token --- rust/Cargo.toml | 2 +- rust/src/interop.rs | 1 + rust/src/interop/lazy_load_token.rs | 116 ++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 rust/src/interop/lazy_load_token.rs diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 17f4eff1..ebfdc290 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -24,7 +24,7 @@ tokio-console = ["dep:console-subscriber", "tokio/tracing"] automerge = "0.7.2" autosurgeon = "0.10.1" futures = "0.3.31" -godot = "0.4.3" +godot = {version = "0.4.5", features = ["experimental-threads"]} tokio = { version = "1.49.0", features = ["full"] } console-subscriber = { version = "0.5.0", optional = true } tracing-subscriber = { version = "0.3.19", features = ["env-filter", "json"] } diff --git a/rust/src/interop.rs b/rust/src/interop.rs index 28a7bbd0..ab6e041b 100644 --- a/rust/src/interop.rs +++ b/rust/src/interop.rs @@ -6,3 +6,4 @@ mod patchwork_config; mod godot_diffs; mod text_differ_view; mod diff_inspector_section; +mod lazy_load_token; \ No newline at end of file diff --git a/rust/src/interop/lazy_load_token.rs b/rust/src/interop/lazy_load_token.rs new file mode 100644 index 00000000..9fd5a2eb --- /dev/null +++ b/rust/src/interop/lazy_load_token.rs @@ -0,0 +1,116 @@ +use crate::{project::branch_db::HistoryRef}; +use godot::{classes::{RefCounted, Resource, ResourceLoader, resource_loader::ThreadLoadStatus}, global, obj::Base, prelude::GodotClass}; +use godot::prelude::*; +use rand::rng; +use samod::DocumentId; + +#[derive(GodotClass, Debug)] +#[class(base=RefCounted)] +pub struct LazyLoadToken { + base: Base, + original_path: Option, + path: String, + resource: Option>, + failed: bool, +} + +#[godot_api] +impl IRefCounted for LazyLoadToken { + fn init(base: Base) -> Self { + Self { + base, + path: String::new(), + original_path: None, + resource: None, + failed: false, + } + } +} + +impl LazyLoadToken { + pub fn new(path: String, original_path: Option) -> Gd { + let mut tok = Self::new_gd(); + tok.bind_mut().set_paths(path, original_path); + tok + } + fn set_paths(&mut self, path: String, original_path: Option) { + self.path = path; + self.original_path = original_path; + } +} + +#[godot_api] +impl LazyLoadToken { + #[func] + fn is_started(&self) -> bool { + if self.failed || self.resource.is_some() { + return true; + } + let status = ResourceLoader::singleton().load_threaded_get_status(&self.path); + if status != ThreadLoadStatus::INVALID_RESOURCE { + return true; + } + false + } + + #[func] + fn is_load_finished(&self) -> bool { + if self.failed || self.resource.is_some() { + return true; + } + let status = ResourceLoader::singleton().load_threaded_get_status(&self.path); + if status == ThreadLoadStatus::LOADED || status == ThreadLoadStatus::FAILED { + return true; + } + false + } + + #[func] + pub fn start_load(&mut self){ + if self.is_started() { + return; + } + if ResourceLoader::singleton().load_threaded_request(&self.path) != global::Error::OK { + self.failed = true; + } + } + + #[func] + pub fn get_resource(&mut self) -> Option> { + if self.resource.is_some() { + return self.resource.clone(); + } + if !self.is_started() { + self.start_load(); + } + if self.failed { + return None; + } + + let res = ResourceLoader::singleton().load_threaded_get(&self.path); + if let Some(mut res) = res { + if let Some(original_path) = self.original_path.as_ref() { + if &res.get_path().to_string() != original_path { + res.set_path_cache(original_path); + } + } + self.resource = Some(res); + } else { + self.failed = true; + } + return self.resource.clone(); + } + + #[func] + pub fn did_fail(&self) -> bool { + self.failed + } + + #[func] + pub fn get_path(&self) -> GString { + if let Some(original_path) = self.original_path.as_ref() { + return GString::from(original_path); + } + GString::from(&self.path) + } +} From 727e5392a399467a4af05840a50571aec8c18c21 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Thu, 29 Jan 2026 17:48:01 -0800 Subject: [PATCH 21/49] refactor diff to not use variants in internal diff structs, use lazy loading for resources --- rust/src/diff/differ.rs | 107 +++++++++++++++++----------- rust/src/diff/resource_differ.rs | 32 +++------ rust/src/diff/scene_differ.rs | 32 +++++---- rust/src/interop/godot_accessors.rs | 9 +++ rust/src/interop/godot_diffs.rs | 34 ++++++--- rust/src/project/driver.rs | 14 ++-- rust/src/project/project.rs | 15 +--- 7 files changed, 141 insertions(+), 102 deletions(-) diff --git a/rust/src/diff/differ.rs b/rust/src/diff/differ.rs index a090a694..6779a5c2 100644 --- a/rust/src/diff/differ.rs +++ b/rust/src/diff/differ.rs @@ -1,16 +1,14 @@ use std::{ cell::RefCell, collections::{HashMap, HashSet}, - path::PathBuf, + path::PathBuf, sync::Arc, }; use automerge::ChangeHash; use godot::{ - builtin::{GString, Variant}, - classes::{ResourceLoader, resource_loader::CacheMode}, - meta::ToGodot, - obj::Singleton, + builtin::{GString, Variant}, classes::{ResourceLoader, resource_loader::CacheMode}, global, meta::ToGodot, obj::Singleton }; +use tokio::sync::Mutex; use tracing::instrument; use crate::{ @@ -56,9 +54,11 @@ pub struct ProjectDiff { } /// Computes diffs between two sets of heads in a project. +#[derive(Debug)] pub struct Differ { - /// Cache that stores our loaded ExtResources so far. - loaded_ext_resources: RefCell>, + /// Cache that keeps track of the original paths of our loaded ExtResources so far. + /// original path -> loaded path + loaded_ext_resources: Arc>>>>>, /// The [BranchDb] we're working off. branch_db: BranchDb, @@ -69,7 +69,7 @@ impl Differ { pub fn new(branch_db: BranchDb) -> Self { Self { branch_db, - loaded_ext_resources: RefCell::new(HashMap::new()), + loaded_ext_resources: Arc::new(Mutex::new(HashMap::new())), } } @@ -79,14 +79,15 @@ impl Differ { path: &String, file_content: &FileContent, ref_: &HistoryRef, - ) -> Option { + ) -> Option { let import_path = format!("{}.import", path); let mut import_file_content = self.get_file_at_ref(&import_path, ref_).await; - // TODO (Lilith): Reimplement this using branchDB - // if import_file_content.is_none() { - // // try at current heads - // import_file_content = self.get_file_at_ref(&import_path, None); - // } + if import_file_content.is_none() { + // try at current heads + if let Some(current_heads) = self.branch_db.get_latest_ref_on_branch(&ref_.branch).await { + import_file_content = self.get_file_at_ref(&import_path, ¤t_heads).await; + } + } return self .create_temp_resource_from_content( &path, @@ -104,7 +105,7 @@ impl Differ { content: &FileContent, temp_id: &String, import_file_content: Option<&FileContent>, - ) -> Option { + ) -> Option { let temp_dir = format!("res://.patchwork/temp_{}/", temp_id); let temp_path = path.replace("res://", &temp_dir); if let Err(e) = content @@ -115,6 +116,7 @@ impl Differ { return None; } + let mut loaded_path: Option = None; if let Some(import_file_content) = import_file_content { if let FileContent::String(import_file_content) = import_file_content { let import_file_content = import_file_content.replace("res://", &temp_dir); @@ -125,23 +127,21 @@ impl Differ { let import_file_path: String = format!("{}.import", temp_path); let _ = FileContent::String(import_file_content).write(&self.branch_db.globalize_path(&import_file_path)); - let res = PatchworkEditorAccessor::import_and_load_resource(&temp_path); - if res.is_nil() { - tracing::error!("error importing resource: {:?}", temp_path); - return None; + let loaded_path_str = PatchworkEditorAccessor::import_and_save_resource_to_temp(&temp_path); + if loaded_path_str.is_empty() { + tracing::error!("error importing resource: {:?}", path); + } else { + loaded_path = Some(loaded_path_str); } - tracing::debug!("successfully imported resource: {:?}", temp_path); - return Some(res); } + } else { + loaded_path = Some(temp_path.clone()); } - let resource = ResourceLoader::singleton() - .load_ex(&GString::from(&temp_path)) - .cache_mode(CacheMode::IGNORE_DEEP) - .done(); - if let Some(resource) = resource { - return Some(resource.to_variant()); - } - None + // let resource = ResourceLoader::singleton() + // .load_ex(&GString::from(&temp_path)) + // .cache_mode(CacheMode::IGNORE_DEEP) + // .done(); + loaded_path } /// Gets the file content at a given path for the specified heads. @@ -177,31 +177,56 @@ impl Differ { } /// Loads an ExtResource given a path, using a cache. - pub(super) async fn load_ext_resource( + pub(super) async fn start_load_ext_resource( &self, path: &String, ref_: &HistoryRef, - ) -> Option { - if let Some(resource) = self.loaded_ext_resources.borrow().get(path) { - return Some(resource.clone()); + content: Option<&FileContent>, + ) -> Option { + + let mut hash_map_insert_guard = Some(self.loaded_ext_resources.lock().await); + if let Some(load_path_tok) = hash_map_insert_guard.as_ref().unwrap().get(&(path.clone(), ref_.clone())).cloned() { + let load_path_tok = load_path_tok.lock().await; + if let Some(load_path) = load_path_tok.as_ref().cloned() { + if ResourceLoader::singleton().load_threaded_request(&load_path) == global::Error::OK { + return Some(load_path); + } + // else we can't load the path; fall-through to re-create the token and resource + } else { + tracing::error!("load path token is None for path: {}", path); + return None; + } + } + // create the token, acquire the lock so that if this is running on multiple threads it doesn't try and create the resource multiple times + let mut load_path_tok = Arc::new(Mutex::new(None)); + let mut mutex_guard = load_path_tok.lock().await; + hash_map_insert_guard.unwrap().insert((path.clone(), ref_.clone()), load_path_tok.clone()); + // release hashmap guard + hash_map_insert_guard = None; + + + + let mut resource_content = None; + if content.is_none() { + resource_content = self.get_file_at_ref(path, ref_).await; } - let resource_content = self.get_file_at_ref(path, ref_).await; - let Some(resource_content) = resource_content else { + let Some(resource_content) = content.or(resource_content.as_ref()) else { return None; }; - let Some(resource) = self - .get_resource_at_ref(path, &resource_content, ref_) + let Some(load_path) = self + .get_resource_at_ref(path, &content.unwrap_or(&resource_content), ref_) .await else { return None; }; - self.loaded_ext_resources - .borrow_mut() - .insert(path.clone(), resource.clone()); - Some(resource) + if ResourceLoader::singleton().load_threaded_request(&load_path) == global::Error::OK { + *mutex_guard = Some(load_path.clone()); + return Some(load_path); + } + None } /// Computes the diff between the two sets of heads. diff --git a/rust/src/diff/resource_differ.rs b/rust/src/diff/resource_differ.rs index caf7079e..936986c0 100644 --- a/rust/src/diff/resource_differ.rs +++ b/rust/src/diff/resource_differ.rs @@ -1,7 +1,7 @@ use godot::builtin::Variant; use crate::{ - diff::differ::{ChangeType, Differ}, fs::file_utils::FileContent, helpers::utils::ToShortForm, project::branch_db::HistoryRef + diff::{differ::{ChangeType, Differ}, scene_differ::VariantValue}, fs::file_utils::FileContent, helpers::utils::ToShortForm, project::branch_db::HistoryRef }; @@ -9,16 +9,16 @@ use crate::{ pub struct BinaryResourceDiff { pub path: String, pub change_type: ChangeType, - pub old_resource: Option, - pub new_resource: Option, + pub old_resource: Option, + pub new_resource: Option, } impl BinaryResourceDiff { pub fn new( path: String, change_type: ChangeType, - old_resource: Option, - new_resource: Option, + old_resource: Option, + new_resource: Option, ) -> BinaryResourceDiff { BinaryResourceDiff { path, @@ -52,22 +52,12 @@ impl Differ { path: &String, content: &FileContent, ref_: &HistoryRef, - ) -> Option { - let import_path = format!("{}.import", path); - let import_file_content = match content { - FileContent::Deleted => None, - _ => self - .get_file_at_ref(&import_path, ref_).await - // TODO (Lilith): make this work - // try at current heads - // .or(self.get_file_at(&import_path, None)), - }; + ) -> Option { - self.create_temp_resource_from_content( - &path, - content, - &ref_.heads.first().to_short_form(), - import_file_content.as_ref(), - ).await + let Some(load_path) = self.start_load_ext_resource(path, ref_, Some(content)).await + else { + return None; + }; + Some(VariantValue::LazyLoadData(path.clone(), load_path)) } } diff --git a/rust/src/diff/scene_differ.rs b/rust/src/diff/scene_differ.rs index 757eb7bd..e280d8e0 100644 --- a/rust/src/diff/scene_differ.rs +++ b/rust/src/diff/scene_differ.rs @@ -124,17 +124,17 @@ pub struct PropertyDiff { /// The change type of the property. pub change_type: ChangeType, /// The old value of the property, if it existed. - pub old_value: Option, + pub old_value: Option, /// The new value of the property, if it still exists. - pub new_value: Option, + pub new_value: Option, } impl PropertyDiff { pub fn new( name: String, change_type: ChangeType, - old_value: Option, - new_value: Option, + old_value: Option, + new_value: Option, ) -> Self { PropertyDiff { name, @@ -158,6 +158,13 @@ enum VariantStrValue { ExtResourceID(String), } + +#[derive(Clone, Debug)] +pub enum VariantValue { + Variant(String), + LazyLoadData(String, String), +} + /// Implement the to_string method for this enum impl std::fmt::Display for VariantStrValue { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -695,19 +702,20 @@ impl Differ { is_script: bool, before: &HistoryRef, after: &HistoryRef, - ) -> Variant { + ) -> VariantValue { // Prevent loading script files during the diff and creating issues for the editor if is_script { - return "