diff --git a/src/agent_data.rs b/src/agent_data.rs index e0d161c..8bd94a3 100644 --- a/src/agent_data.rs +++ b/src/agent_data.rs @@ -1,7 +1,7 @@ use gethostname::gethostname; -use log::info; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use tracing::info; #[derive(Debug, Deserialize, Serialize, Clone)] struct AgentVersion { diff --git a/src/file_info.rs b/src/file_info.rs index 1d5fa9a..b89e8d7 100644 --- a/src/file_info.rs +++ b/src/file_info.rs @@ -1,8 +1,16 @@ -use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, Value, ValueRef}; -use rusqlite::ToSql; +use rusqlite::{ + types::{FromSql, FromSqlError, FromSqlResult, Value, ValueRef}, + ToSql, +}; use serde::{Deserialize, Serialize}; -use std::path::PathBuf; -use std::time::SystemTime; +use std::{ + fs, + io::Read, + path::{Path, PathBuf}, + time::SystemTime, +}; +use tracing::warn; +use xxhash_rust::xxh3::xxh3_128; #[derive(Debug, Serialize, Deserialize, Clone)] pub struct FileInfo { @@ -79,3 +87,97 @@ impl FromSql for TidyScore { } } } + +#[cfg(not(target_os = "windows"))] +pub fn fix_canonicalize_path>(path: P) -> PathBuf { + path.as_ref().into() +} + +#[cfg(target_os = "windows")] +pub fn fix_canonicalize_path>(path: P) -> PathBuf { + const UNCPREFIX: &str = r"\\?\"; + let p: String = path.as_ref().display().to_string(); + if p.starts_with(UNCPREFIX) { + p[UNCPREFIX.len()..].into() + } else { + p.into() + } +} + +pub fn get_file_signature(path: &PathBuf) -> u128 { + let mut file = fs::File::open(path).unwrap(); + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).unwrap(); + xxh3_128(&buffer) +} + +pub fn create_file_info(path: &PathBuf) -> Option { + match fs::metadata(path) { + Ok(md) => { + let size: u64 = md.len(); + let last_modified: SystemTime = md.modified().ok()?; + let last_accessed: SystemTime = md.accessed().ok()?; + let file_signature = get_file_signature(path); + + Some(FileInfo { + name: Path::new(path.to_str()?).file_name()?.to_str()?.to_owned(), + path: path.clone(), + size, + hash: Some(file_signature.to_string()), + last_modified, + last_accessed, + ..Default::default() + }) + } + Err(err) => { + warn!("Could not get access to {:?} metadata: {}", path, err); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[cfg(not(target_os = "windows"))] + fn test_fix_canonicalize_path_unix() { + let path = "tests/assets/test_folder"; + let canonicalized = fix_canonicalize_path(path); + assert_eq!(canonicalized, PathBuf::from(path)); + } + + #[test] + #[cfg(target_os = "windows")] + fn test_fix_canonicalize_path_windows() { + let path = r"C:\tests\assets\test_folder"; + let canonicalized = fix_canonicalize_path(path); + assert_eq!(canonicalized, PathBuf::from(path)); + } + + #[test] + fn test_get_file_signature() { + let path: PathBuf = [r"tests", r"assets", r"test_folder", r"test-file-1"] + .iter() + .collect(); + let hash = get_file_signature(&path); + assert_eq!(hash, 53180848542178601830765469314885156230); + } + + #[test] + fn test_create_file_info() { + let path: PathBuf = [r"tests", r"assets", r"test_folder", r"test-file-1"] + .iter() + .collect(); + if let Some(file_info) = create_file_info(&path) { + assert_eq!(file_info.path, path); + assert_eq!(file_info.size, 100); + if let Some(hash) = file_info.hash { + assert_eq!(hash, "53180848542178601830765469314885156230"); + } + assert_ne!(file_info.last_modified, SystemTime::UNIX_EPOCH); + assert_ne!(file_info.last_accessed, SystemTime::UNIX_EPOCH); + } + } +} diff --git a/src/file_lister.rs b/src/file_lister.rs index 9bfc6ae..1ba63f3 100644 --- a/src/file_lister.rs +++ b/src/file_lister.rs @@ -1,16 +1,6 @@ -use crate::file_info::FileInfo; +use crate::file_info::{create_file_info, fix_canonicalize_path, FileInfo}; use std::fs; -use std::io::Read; -use std::path::{Path, PathBuf}; -use std::time::SystemTime; -use xxhash_rust::xxh3::xxh3_128; - -fn get_file_signature(path: &PathBuf) -> u128 { - let mut file = fs::File::open(path).unwrap(); - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer).unwrap(); - xxh3_128(&buffer) -} +use std::path::PathBuf; pub fn list_directories(directories: Vec) -> Result, std::io::Error> { let mut files: Vec = Vec::new(); @@ -19,30 +9,14 @@ pub fn list_directories(directories: Vec) -> Result, std: if directory.is_dir() { for entry in fs::read_dir(&directory)? { let entry: fs::DirEntry = entry?; - let path: PathBuf = entry.path(); + let path: PathBuf = fix_canonicalize_path(fs::canonicalize(entry.path())?); if path.is_dir() { files.extend(list_directories(vec![path])?); - } else if let Some(file) = path.to_str() { - let md: fs::Metadata = fs::metadata(&path)?; - let size: u64 = md.len(); - let last_modified: SystemTime = md.modified()?; - let last_accessed: SystemTime = md.accessed()?; - let file_signature = get_file_signature(&path); - files.push(FileInfo { - name: Path::new(file) - .file_name() - .unwrap() - .to_str() - .unwrap() - .to_owned(), - path, - size, - hash: Some(file_signature.to_string()), - last_modified, - last_accessed, - ..Default::default() - }); + } else if path.to_str().is_some() { + if let Some(file_info) = create_file_info(&path) { + files.push(file_info); + } } } } diff --git a/src/file_watcher.rs b/src/file_watcher.rs index 52e86c2..270907a 100644 --- a/src/file_watcher.rs +++ b/src/file_watcher.rs @@ -1,13 +1,15 @@ -use log::error; use notify::RecursiveMode::Recursive as RecursiveWatcher; use notify::Watcher; +use std::path::PathBuf; +use std::sync::mpsc; use std::time; +use tracing::error; pub fn watch_directories( - directories: Vec, + directories: Vec, sender: crossbeam_channel::Sender, ) { - let (tx, rx) = std::sync::mpsc::channel(); + let (tx, rx) = mpsc::channel(); let mut debouncer: notify_debouncer_full::Debouncer< notify::RecommendedWatcher, @@ -21,10 +23,23 @@ pub fn watch_directories( }; for directory in directories { - if let Err(err) = debouncer.watcher().watch(&directory, RecursiveWatcher) { - error!("{:?}: {:?}", directory, err); + let clean_directory = match directory.canonicalize() { + Ok(clean_directory) => clean_directory, + Err(err) => { + error!("error with {:?}: {:?}", directory, err); + continue; + } + }; + + if let Err(err) = debouncer + .watcher() + .watch(&clean_directory, RecursiveWatcher) + { + error!("{:?}: {:?}", clean_directory, err); } else { - debouncer.cache().add_root(&directory, RecursiveWatcher); + debouncer + .cache() + .add_root(&clean_directory, RecursiveWatcher); } } diff --git a/src/http_server.rs b/src/http_server.rs index 2b1e031..b7a842e 100644 --- a/src/http_server.rs +++ b/src/http_server.rs @@ -5,7 +5,6 @@ use crate::my_files; use crate::my_files::{ConfigurationPresent, ConnectionManagerPresent, Sealed}; use axum::{extract::Query, extract::State, routing::get, Json, Router}; use lazy_static::lazy_static; -use log::{error, info}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::net::SocketAddr; @@ -13,7 +12,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use tokio::net::TcpListener; use tower_http::trace::{self, TraceLayer}; -use tracing::Level; +use tracing::{error, info, Level}; lazy_static! { static ref AGENT_LOGGING_LEVEL: HashMap = { diff --git a/src/lib.rs b/src/lib.rs index b0a7be4..8a05b24 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,8 +9,9 @@ mod tidy_algo; use crate::tidy_algo::tidy_algo::TidyAlgo; use http_server::HttpServerBuilder; -use log::{debug, error, info}; +use notify::EventKind; use std::{path::PathBuf, thread}; +use tracing::{error, info}; pub async fn run() { match std::env::var("TIDY_BACKTRACE") { @@ -40,25 +41,12 @@ pub async fn run() { info!("MyFilesDB sucessfully initialized"); let mut tidy_algo = TidyAlgo::new(); + let basic_ruleset_path: PathBuf = [r"config", r"rules", r"basic.yml"].iter().collect(); info!("TidyAlgo sucessfully created"); - tidy_algo.load_rules_from_file(&my_files, PathBuf::from("config/rules/basic.yml")); + tidy_algo.load_rules_from_file(&my_files, basic_ruleset_path); info!("TidyAlgo sucessfully loaded rules from config/rules/basic.yml"); - match file_lister::list_directories(config.file_lister_config.dir) { - Ok(files_vec) => { - for file in &files_vec { - match my_files.add_file_to_db(file) { - Ok(_) => {} - Err(error) => { - error!("{:?}", error); - } - } - } - } - Err(error) => { - error!("{}", error); - } - } + list_directories(config.file_lister_config.dir, &my_files); let server = HttpServerBuilder::new() .my_files_builder(my_files_builder) @@ -76,14 +64,57 @@ pub async fn run() { }); info!("HTTP Server Started"); - let (sender, receiver) = crossbeam_channel::unbounded(); - let watch_directories_thread: thread::JoinHandle<()> = thread::spawn(move || { - file_watcher::watch_directories(config.file_watcher_config.dir.clone(), sender); + let (file_watcher_sender, file_watcher_receiver) = crossbeam_channel::unbounded(); + let file_watcher_thread: thread::JoinHandle<()> = thread::spawn(move || { + file_watcher::watch_directories( + config.file_watcher_config.dir.clone(), + file_watcher_sender, + ); }); info!("File Events Watcher Started"); - for event in receiver { - debug!("{:?}", event); + for file_watcher_event in file_watcher_receiver { + handle_file_events(&file_watcher_event, &my_files); } - watch_directories_thread.join().unwrap(); + file_watcher_thread.join().unwrap(); +} + +fn list_directories(config: Vec, my_files: &my_files::MyFiles) { + match file_lister::list_directories(config) { + Ok(files_vec) => { + for file in &files_vec { + match my_files.add_file_to_db(file) { + Ok(_) => {} + Err(error) => { + error!("{:?}", error); + } + } + } + } + Err(error) => { + error!("{}", error); + } + } +} + +fn handle_file_events(event: ¬ify::Event, my_files: &my_files::MyFiles) { + info!("event: kind: {:?}\tpaths: {:?}", event.kind, &event.paths); + + if let EventKind::Remove(_) = event.kind { + match my_files.remove_file_from_db(event.paths[0].clone()) { + Ok(_) => {} + Err(error) => { + error!("{:?}", error); + } + } + } else if let EventKind::Create(_) = event.kind { + if let Some(file) = file_info::create_file_info(&event.paths[0].clone()) { + match my_files.add_file_to_db(&file) { + Ok(_) => {} + Err(error) => { + error!("{:?}", error); + } + } + } + } } diff --git a/src/my_files.rs b/src/my_files.rs index 27d64d5..171f9f0 100644 --- a/src/my_files.rs +++ b/src/my_files.rs @@ -3,12 +3,12 @@ use crate::file_info::{FileInfo, TidyScore}; use chrono::{DateTime, Utc}; use core::marker::PhantomData; use itertools::{Either, Itertools}; -use log::{error, info, warn}; use r2d2::{Pool, PooledConnection}; use r2d2_sqlite::SqliteConnectionManager; use rusqlite::{params, Result, ToSql}; use std::path; use std::path::PathBuf; +use tracing::{error, info, warn}; // region: --- MyFiles builder states #[derive(Default, Clone)] @@ -169,21 +169,25 @@ impl MyFiles { } } } - pub fn remove_file_from_db(&self, file_path: &str) -> Result<()> { - match self - .connection_pool - .execute("DELETE FROM my_files WHERE path = ?1", params![file_path]) - { + + pub fn remove_file_from_db(&self, file_path: PathBuf) -> Result<()> { + let str_filepath = file_path.to_str().unwrap(); + + match self.connection_pool.execute( + "DELETE FROM my_files WHERE path = ?1", + params![str_filepath], + ) { Ok(_) => { - info!("{} removed from my_files", file_path); + info!("{} removed from my_files", str_filepath); Ok(()) } Err(error) => { - error!("Error removing {} from my_files: {}", file_path, error); + error!("Error removing {} from my_files: {}", str_filepath, error); Err(error) } } } + pub fn add_file_to_db(&self, file: &FileInfo) -> Result { let last_modified: DateTime = file.last_modified.into(); let last_accessed: DateTime = file.last_accessed.into(); @@ -541,6 +545,8 @@ impl MyFiles { #[cfg(test)] mod tests { + use std::env::current_dir; + use super::*; use crate::{configuration, file_lister}; @@ -612,20 +618,18 @@ mod tests { duplicated: None, unused: true, }; + let mut tests_dir = current_dir().unwrap(); + tests_dir.push( + [r"tests", "assets", "test_folder"] + .iter() + .collect::(), + ); + my_files - .set_tidyscore( - [r"tests", "assets", "test_folder", "test-file-1"] - .iter() - .collect(), - &dummy_score, - ) + .set_tidyscore(tests_dir.join("test-file-1"), &dummy_score) .unwrap(); let mut score = my_files - .get_tidyscore( - [r"tests", "assets", "test_folder", "test-file-1"] - .iter() - .collect(), - ) + .get_tidyscore(tests_dir.join("test-file-1")) .unwrap(); let is_misnamed = score.misnamed; let is_unused = score.unused; @@ -633,41 +637,16 @@ mod tests { assert!(is_unused); my_files - .add_duplicated_file_to_db( - [r"tests", "assets", "test_folder", "test-file-1"] - .iter() - .collect(), - [r"tests", "assets", "test_folder", "test-file-1-dup-1"] - .iter() - .collect(), - ) + .add_duplicated_file_to_db(tests_dir.join("test-file-1"), tests_dir.join("test-file-2")) .unwrap(); my_files - .add_duplicated_file_to_db( - [r"tests", "assets", "test_folder", "test-file-1"] - .iter() - .collect(), - [r"tests", "assets", "test_folder", "test-file-1-dup-2"] - .iter() - .collect(), - ) + .add_duplicated_file_to_db(tests_dir.join("test-file-1"), tests_dir.join("test-file-3")) .unwrap(); my_files - .add_duplicated_file_to_db( - [r"tests", "assets", "test_folder", "test-file-1"] - .iter() - .collect(), - [r"tests", "assets", "test_folder", "test-file-1-dup-3"] - .iter() - .collect(), - ) + .add_duplicated_file_to_db(tests_dir.join("test-file-1"), tests_dir.join("test-file-4")) .unwrap(); score = my_files - .get_tidyscore( - [r"tests", "assets", "test_folder", "test-file-1"] - .iter() - .collect(), - ) + .get_tidyscore(tests_dir.join("test-file-1")) .unwrap(); let is_duplicated = match score.duplicated { Some(duplicated) => !duplicated.is_empty(), diff --git a/src/tidy_algo/tidy_algo.rs b/src/tidy_algo/tidy_algo.rs index 20aefe3..f78b16b 100644 --- a/src/tidy_algo/tidy_algo.rs +++ b/src/tidy_algo/tidy_algo.rs @@ -4,9 +4,9 @@ use crate::tidy_algo::tidy_rules::duplicated::duplicated; use crate::tidy_algo::tidy_rules::misnamed::misnamed; use crate::tidy_algo::tidy_rules::perished::perished; use config::{Config, ConfigError, File, Value}; -use log::debug; use std::collections::HashMap; use std::path; +use tracing::debug; /// Represents a rule that can be applied to a file #[allow(dead_code)] diff --git a/src/tidy_algo/tidy_rules/duplicated.rs b/src/tidy_algo/tidy_rules/duplicated.rs index 4e019de..7a73aaa 100644 --- a/src/tidy_algo/tidy_rules/duplicated.rs +++ b/src/tidy_algo/tidy_rules/duplicated.rs @@ -1,7 +1,6 @@ -use std::collections::HashMap; - use config::Value; -use log::error; +use std::collections::HashMap; +use tracing::error; use crate::{ file_info::{FileInfo, TidyScore}, diff --git a/src/tidy_algo/tidy_rules/misnamed.rs b/src/tidy_algo/tidy_rules/misnamed.rs index 7ff3ee7..a3d0a8f 100644 --- a/src/tidy_algo/tidy_rules/misnamed.rs +++ b/src/tidy_algo/tidy_rules/misnamed.rs @@ -1,7 +1,7 @@ use config::Value; -use log::warn; use regex::Regex; use std::collections::HashMap; +use tracing::warn; use crate::{ file_info::{FileInfo, TidyScore}, diff --git a/src/tidy_algo/tidy_rules/mod.rs b/src/tidy_algo/tidy_rules/mod.rs index 56e4cd7..4b8e900 100644 --- a/src/tidy_algo/tidy_rules/mod.rs +++ b/src/tidy_algo/tidy_rules/mod.rs @@ -1,3 +1,3 @@ pub mod duplicated; pub mod misnamed; -pub mod perished; \ No newline at end of file +pub mod perished; diff --git a/src/tidy_algo/tidy_rules/perished.rs b/src/tidy_algo/tidy_rules/perished.rs index c4bcd83..67671bf 100644 --- a/src/tidy_algo/tidy_rules/perished.rs +++ b/src/tidy_algo/tidy_rules/perished.rs @@ -1,7 +1,7 @@ use config::Value; use lazy_static::lazy_static; -use log::warn; use std::collections::HashMap; +use tracing::warn; use crate::{ file_info::{FileInfo, TidyScore},