Merge pull request #119 from TidyBee/109-update-the-database-when-an-…

…event-occurs-on-the-fs

109 update the database when an event occurs on the fs

src/agent_data.rs

@@ -1,7 +1,7 @@
 use gethostname::gethostname;
-use log::info;
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
+use tracing::info;
 
 #[derive(Debug, Deserialize, Serialize, Clone)]
 struct AgentVersion {

src/file_info.rs

@@ -1,8 +1,16 @@
-use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, Value, ValueRef};
-use rusqlite::ToSql;
+use rusqlite::{
+    types::{FromSql, FromSqlError, FromSqlResult, Value, ValueRef},
+    ToSql,
+};
 use serde::{Deserialize, Serialize};
-use std::path::PathBuf;
-use std::time::SystemTime;
+use std::{
+    fs,
+    io::Read,
+    path::{Path, PathBuf},
+    time::SystemTime,
+};
+use tracing::warn;
+use xxhash_rust::xxh3::xxh3_128;
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct FileInfo {
@@ -79,3 +87,97 @@ impl FromSql for TidyScore {
         }
     }
 }
+
+#[cfg(not(target_os = "windows"))]
+pub fn fix_canonicalize_path<P: AsRef<Path>>(path: P) -> PathBuf {
+    path.as_ref().into()
+}
+
+#[cfg(target_os = "windows")]
+pub fn fix_canonicalize_path<P: AsRef<Path>>(path: P) -> PathBuf {
+    const UNCPREFIX: &str = r"\\?\";
+    let p: String = path.as_ref().display().to_string();
+    if p.starts_with(UNCPREFIX) {
+        p[UNCPREFIX.len()..].into()
+    } else {
+        p.into()
+    }
+}
+
+pub fn get_file_signature(path: &PathBuf) -> u128 {
+    let mut file = fs::File::open(path).unwrap();
+    let mut buffer = Vec::new();
+    file.read_to_end(&mut buffer).unwrap();
+    xxh3_128(&buffer)
+}
+
+pub fn create_file_info(path: &PathBuf) -> Option<FileInfo> {
+    match fs::metadata(path) {
+        Ok(md) => {
+            let size: u64 = md.len();
+            let last_modified: SystemTime = md.modified().ok()?;
+            let last_accessed: SystemTime = md.accessed().ok()?;
+            let file_signature = get_file_signature(path);
+
+            Some(FileInfo {
+                name: Path::new(path.to_str()?).file_name()?.to_str()?.to_owned(),
+                path: path.clone(),
+                size,
+                hash: Some(file_signature.to_string()),
+                last_modified,
+                last_accessed,
+                ..Default::default()
+            })
+        }
+        Err(err) => {
+            warn!("Could not get access to {:?} metadata: {}", path, err);
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[cfg(not(target_os = "windows"))]
+    fn test_fix_canonicalize_path_unix() {
+        let path = "tests/assets/test_folder";
+        let canonicalized = fix_canonicalize_path(path);
+        assert_eq!(canonicalized, PathBuf::from(path));
+    }
+
+    #[test]
+    #[cfg(target_os = "windows")]
+    fn test_fix_canonicalize_path_windows() {
+        let path = r"C:\tests\assets\test_folder";
+        let canonicalized = fix_canonicalize_path(path);
+        assert_eq!(canonicalized, PathBuf::from(path));
+    }
+
+    #[test]
+    fn test_get_file_signature() {
+        let path: PathBuf = [r"tests", r"assets", r"test_folder", r"test-file-1"]
+            .iter()
+            .collect();
+        let hash = get_file_signature(&path);
+        assert_eq!(hash, 53180848542178601830765469314885156230);
+    }
+
+    #[test]
+    fn test_create_file_info() {
+        let path: PathBuf = [r"tests", r"assets", r"test_folder", r"test-file-1"]
+            .iter()
+            .collect();
+        if let Some(file_info) = create_file_info(&path) {
+            assert_eq!(file_info.path, path);
+            assert_eq!(file_info.size, 100);
+            if let Some(hash) = file_info.hash {
+                assert_eq!(hash, "53180848542178601830765469314885156230");
+            }
+            assert_ne!(file_info.last_modified, SystemTime::UNIX_EPOCH);
+            assert_ne!(file_info.last_accessed, SystemTime::UNIX_EPOCH);
+        }
+    }
+}

src/file_lister.rs

@@ -1,32 +1,6 @@
-use crate::file_info::FileInfo;
+use crate::file_info::{create_file_info, fix_canonicalize_path, FileInfo};
 use std::fs;
-use std::io::Read;
-use std::path::{Path, PathBuf};
-use std::time::SystemTime;
-use xxhash_rust::xxh3::xxh3_128;
-
-fn get_file_signature(path: &PathBuf) -> u128 {
-    let mut file = fs::File::open(path).unwrap();
-    let mut buffer = Vec::new();
-    file.read_to_end(&mut buffer).unwrap();
-    xxh3_128(&buffer)
-}
-
-#[cfg(not(target_os = "windows"))]
-fn fix_canonicalize_path<P: AsRef<Path>>(path: P) -> PathBuf {
-    path.as_ref().into()
-}
-
-#[cfg(target_os = "windows")]
-fn fix_canonicalize_path<P: AsRef<Path>>(path: P) -> PathBuf {
-    const UNCPREFIX: &str = r"\\?\";
-    let p: String = path.as_ref().display().to_string();
-    if p.starts_with(UNCPREFIX) {
-        p[UNCPREFIX.len()..].into()
-    } else {
-        p.into()
-    }
-}
+use std::path::PathBuf;
 
 pub fn list_directories(directories: Vec<PathBuf>) -> Result<Vec<FileInfo>, std::io::Error> {
     let mut files: Vec<FileInfo> = Vec::new();
@@ -39,26 +13,10 @@ pub fn list_directories(directories: Vec<PathBuf>) -> Result<Vec<FileInfo>, std:
 
                 if path.is_dir() {
                     files.extend(list_directories(vec![path])?);
-                } else if let Some(file) = path.to_str() {
-                    let md: fs::Metadata = fs::metadata(&path)?;
-                    let size: u64 = md.len();
-                    let last_modified: SystemTime = md.modified()?;
-                    let last_accessed: SystemTime = md.accessed()?;
-                    let file_signature = get_file_signature(&path);
-                    files.push(FileInfo {
-                        name: Path::new(file)
-                            .file_name()
-                            .unwrap()
-                            .to_str()
-                            .unwrap()
-                            .to_owned(),
-                        path,
-                        size,
-                        hash: Some(file_signature.to_string()),
-                        last_modified,
-                        last_accessed,
-                        ..Default::default()
-                    });
+                } else if path.to_str().is_some() {
+                    if let Some(file_info) = create_file_info(&path) {
+                        files.push(file_info);
+                    }
                 }
             }
         }

src/file_watcher.rs

@@ -1,13 +1,15 @@
-use log::error;
 use notify::RecursiveMode::Recursive as RecursiveWatcher;
 use notify::Watcher;
+use std::path::PathBuf;
+use std::sync::mpsc;
 use std::time;
+use tracing::error;
 
 pub fn watch_directories(
-    directories: Vec<std::path::PathBuf>,
+    directories: Vec<PathBuf>,
     sender: crossbeam_channel::Sender<notify_debouncer_full::DebouncedEvent>,
 ) {
-    let (tx, rx) = std::sync::mpsc::channel();
+    let (tx, rx) = mpsc::channel();
 
     let mut debouncer: notify_debouncer_full::Debouncer<
         notify::RecommendedWatcher,
@@ -21,10 +23,23 @@ pub fn watch_directories(
     };
 
     for directory in directories {
-        if let Err(err) = debouncer.watcher().watch(&directory, RecursiveWatcher) {
-            error!("{:?}: {:?}", directory, err);
+        let clean_directory = match directory.canonicalize() {
+            Ok(clean_directory) => clean_directory,
+            Err(err) => {
+                error!("error with {:?}: {:?}", directory, err);
+                continue;
+            }
+        };
+
+        if let Err(err) = debouncer
+            .watcher()
+            .watch(&clean_directory, RecursiveWatcher)
+        {
+            error!("{:?}: {:?}", clean_directory, err);
         } else {
-            debouncer.cache().add_root(&directory, RecursiveWatcher);
+            debouncer
+                .cache()
+                .add_root(&clean_directory, RecursiveWatcher);
         }
     }
 

src/http_server.rs

@@ -5,15 +5,14 @@ use crate::my_files;
 use crate::my_files::{ConfigurationPresent, ConnectionManagerPresent, Sealed};
 use axum::{extract::Query, extract::State, routing::get, Json, Router};
 use lazy_static::lazy_static;
-use log::{error, info};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex};
 use tokio::net::TcpListener;
 use tower_http::trace::{self, TraceLayer};
-use tracing::Level;
+use tracing::{error, info, Level};
 
 lazy_static! {
     static ref AGENT_LOGGING_LEVEL: HashMap<String, Level> = {

src/lib.rs

@@ -9,8 +9,9 @@ mod tidy_algo;
 
 use crate::tidy_algo::tidy_algo::TidyAlgo;
 use http_server::HttpServerBuilder;
-use log::{debug, error, info};
+use notify::EventKind;
 use std::{path::PathBuf, thread};
+use tracing::{error, info};
 
 pub async fn run() {
     match std::env::var("TIDY_BACKTRACE") {
@@ -40,25 +41,12 @@ pub async fn run() {
     info!("MyFilesDB sucessfully initialized");
 
     let mut tidy_algo = TidyAlgo::new();
+    let basic_ruleset_path: PathBuf = [r"config", r"rules", r"basic.yml"].iter().collect();
     info!("TidyAlgo sucessfully created");
-    tidy_algo.load_rules_from_file(&my_files, PathBuf::from("config/rules/basic.yml"));
+    tidy_algo.load_rules_from_file(&my_files, basic_ruleset_path);
     info!("TidyAlgo sucessfully loaded rules from config/rules/basic.yml");
 
-    match file_lister::list_directories(config.file_lister_config.dir) {
-        Ok(files_vec) => {
-            for file in &files_vec {
-                match my_files.add_file_to_db(file) {
-                    Ok(_) => {}
-                    Err(error) => {
-                        error!("{:?}", error);
-                    }
-                }
-            }
-        }
-        Err(error) => {
-            error!("{}", error);
-        }
-    }
+    list_directories(config.file_lister_config.dir, &my_files);
 
     let server = HttpServerBuilder::new()
         .my_files_builder(my_files_builder)
@@ -76,14 +64,57 @@ pub async fn run() {
     });
     info!("HTTP Server Started");
 
-    let (sender, receiver) = crossbeam_channel::unbounded();
-    let watch_directories_thread: thread::JoinHandle<()> = thread::spawn(move || {
-        file_watcher::watch_directories(config.file_watcher_config.dir.clone(), sender);
+    let (file_watcher_sender, file_watcher_receiver) = crossbeam_channel::unbounded();
+    let file_watcher_thread: thread::JoinHandle<()> = thread::spawn(move || {
+        file_watcher::watch_directories(
+            config.file_watcher_config.dir.clone(),
+            file_watcher_sender,
+        );
     });
     info!("File Events Watcher Started");
-    for event in receiver {
-        debug!("{:?}", event);
+    for file_watcher_event in file_watcher_receiver {
+        handle_file_events(&file_watcher_event, &my_files);
     }
 
-    watch_directories_thread.join().unwrap();
+    file_watcher_thread.join().unwrap();
+}
+
+fn list_directories(config: Vec<PathBuf>, my_files: &my_files::MyFiles) {
+    match file_lister::list_directories(config) {
+        Ok(files_vec) => {
+            for file in &files_vec {
+                match my_files.add_file_to_db(file) {
+                    Ok(_) => {}
+                    Err(error) => {
+                        error!("{:?}", error);
+                    }
+                }
+            }
+        }
+        Err(error) => {
+            error!("{}", error);
+        }
+    }
+}
+
+fn handle_file_events(event: &notify::Event, my_files: &my_files::MyFiles) {
+    info!("event: kind: {:?}\tpaths: {:?}", event.kind, &event.paths);
+
+    if let EventKind::Remove(_) = event.kind {
+        match my_files.remove_file_from_db(event.paths[0].clone()) {
+            Ok(_) => {}
+            Err(error) => {
+                error!("{:?}", error);
+            }
+        }
+    } else if let EventKind::Create(_) = event.kind {
+        if let Some(file) = file_info::create_file_info(&event.paths[0].clone()) {
+            match my_files.add_file_to_db(&file) {
+                Ok(_) => {}
+                Err(error) => {
+                    error!("{:?}", error);
+                }
+            }
+        }
+    }
 }

src/my_files.rs

@@ -3,12 +3,12 @@ use crate::file_info::{FileInfo, TidyScore};
 use chrono::{DateTime, Utc};
 use core::marker::PhantomData;
 use itertools::{Either, Itertools};
-use log::{error, info, warn};
 use r2d2::{Pool, PooledConnection};
 use r2d2_sqlite::SqliteConnectionManager;
 use rusqlite::{params, Result, ToSql};
 use std::path;
 use std::path::PathBuf;
+use tracing::{error, info, warn};
 
 // region: --- MyFiles builder states
 #[derive(Default, Clone)]
@@ -619,7 +619,11 @@ mod tests {
             unused: true,
         };
         let mut tests_dir = current_dir().unwrap();
-        tests_dir.push([r"tests", "assets", "test_folder"].iter().collect::<PathBuf>());
+        tests_dir.push(
+            [r"tests", "assets", "test_folder"]
+                .iter()
+                .collect::<PathBuf>(),
+        );
 
         my_files
             .set_tidyscore(tests_dir.join("test-file-1"), &dummy_score)

src/tidy_algo/tidy_algo.rs

@@ -4,9 +4,9 @@ use crate::tidy_algo::tidy_rules::duplicated::duplicated;
 use crate::tidy_algo::tidy_rules::misnamed::misnamed;
 use crate::tidy_algo::tidy_rules::perished::perished;
 use config::{Config, ConfigError, File, Value};
-use log::debug;
 use std::collections::HashMap;
 use std::path;
+use tracing::debug;
 
 /// Represents a rule that can be applied to a file
 #[allow(dead_code)]