diff --git a/core/Cargo.toml b/core/Cargo.toml index 5826e8d..57470fe 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -2,7 +2,7 @@ name = "hyperparameter" version = "0.5.9" license = "Apache-2.0" -description = "A high performance configuration system for Rust and Python." +description = "A high performance configuration system for Rust." homepage = "https://reiase.github.io/hyperparameter/" readme = "examples/rust/README.md" repository = "https://github.com/reiase/hyperparameter" @@ -16,10 +16,9 @@ exclude = [ ] [features] -default = ["json", "toml", "ini", "clap"] +default = ["json", "toml", "clap"] json = ["config/json"] toml = ["config/toml"] -ini = ["config/ini"] clap = ["dep:linkme", "dep:clap"] [lib] @@ -27,11 +26,8 @@ name = "hyperparameter" crate-type = ["rlib", "staticlib"] [dependencies] -backtrace = "0.3.69" lazy_static = "1.4.0" phf = { version = "0.11", features = ["macros"] } -signal-hook = "0.3.17" -tokio = { version = "1.31.0", features = ["full"] } xxhash-rust = { version = "0.8.7", features = ["xxh3", "xxh64", "const_xxh64"] } const-str = "0.5.6" config = "0.14.0" @@ -46,18 +42,10 @@ clap = { version = "4.4.7", features = ["derive"] } [profile.dev] overflow-checks = false +[profile.release] +strip = "debuginfo" +lto = true + [[bench]] name = "bench_apis" harness = false - -[[example]] -name = "clap_mini" -path = "examples/rust/clap_mini.rs" - -[[example]] -name = "clap_layered" -path = "examples/rust/clap_layered.rs" - -[[example]] -name = "clap_full" -path = "examples/rust/clap_full.rs" diff --git a/core/benches/bench_apis.rs b/core/benches/bench_apis.rs index 58ac852..e5161c1 100644 --- a/core/benches/bench_apis.rs +++ b/core/benches/bench_apis.rs @@ -27,7 +27,7 @@ fn foo_with_config(x: i64, cfg: &Config) -> i64 { fn call_foo(nloop: i64) -> i64 { let mut sum = 0; for i in 0..nloop { - sum = sum + foo(i, 42); + sum += foo(i, 42); } sum } @@ -39,7 +39,7 @@ fn call_foo_with_ps(nloop: i64) -> i64 { with_params! { set y = 42; - sum = sum + foo_with_ps(i); + sum += foo_with_ps(i); } } sum @@ -53,7 +53,7 @@ fn call_foo_with_ps_optimized(nloop: i64) -> i64 { set y = 42; for i in 0..nloop { - sum = sum + foo_with_ps(i); + sum += foo_with_ps(i); } } sum @@ -62,12 +62,12 @@ fn call_foo_with_ps_optimized(nloop: i64) -> i64 { #[inline(never)] fn call_foo_with_ps_and_raw_btree(nloop: i64) -> i64 { let mut sum = 0; - const KEY: u64 = xxh::xxhash("y".as_bytes()); + const KEY: u64 = xxhash("y".as_bytes()); with_params! { set y = 42; for i in 0..nloop { - sum = sum + THREAD_STORAGE.with(|ts| ts.borrow_mut().get_or_else(KEY, i)); + sum += THREAD_STORAGE.with(|ts| ts.borrow_mut().get_or_else(KEY, i)); } } sum @@ -77,7 +77,7 @@ fn call_foo_with_ps_and_raw_btree(nloop: i64) -> i64 { fn call_foo_with_config_rs(nloop: i64, cfg: &Config) -> i64 { let mut sum = 0; for i in 0..nloop { - sum = sum + foo_with_config(i, cfg); + sum += foo_with_config(i, cfg); } sum } @@ -106,12 +106,12 @@ pub fn bench_apis_with_ps(c: &mut Criterion) { pub fn bench_config_rs(c: &mut Criterion) { let cfg = config::Config::builder() - .add_source(config::File::from_str( - "{\"y\": 1}", - config::FileFormat::Json, - )) - .build() - .unwrap(); + .add_source(config::File::from_str( + "{\"y\": 1}", + config::FileFormat::Json, + )) + .build() + .unwrap(); c.bench_function("raw api with config-rs", |b| { b.iter(|| call_foo_with_config_rs(black_box(10000), &cfg)) }); @@ -123,5 +123,6 @@ criterion_group!( bench_apis_with_ps_and_raw_btree, bench_apis_with_ps_optimized, bench_apis_with_ps, + bench_config_rs, ); criterion_main!(benches); diff --git a/core/examples/clap_full.rs b/core/examples/clap_full.rs index 91f68d4..35924a5 100644 --- a/core/examples/clap_full.rs +++ b/core/examples/clap_full.rs @@ -12,14 +12,14 @@ struct CommandLineArgs { define: Vec, /// Specifies the configuration file path. - #[arg(short = 'C', long, default_value = "examples/rust/cfg.toml")] + #[arg(short = 'C', long, default_value = "examples/cfg.toml")] config: String, } fn foo(desc: &str) { with_params! { /// Example param1 // this explain shows in ` --help` - get param1 = example.param1 or "default".to_string(); + get param1 = example.param1 or "default".to_string(); println!("param1={} // {}", param1, desc); } @@ -30,7 +30,8 @@ fn main() { let config_path = Path::new(&args.config); let config = config::Config::builder() .add_source(File::from(config_path)) - .build().unwrap(); + .build() + .unwrap(); foo("Outside any specific scope"); diff --git a/core/examples/clap_layered.rs b/core/examples/clap_layered.rs index b852d25..b0ce9fa 100644 --- a/core/examples/clap_layered.rs +++ b/core/examples/clap_layered.rs @@ -12,7 +12,7 @@ struct CommandLineArgs { define: Vec, /// Specifies the configuration file path. - #[arg(short = 'C', long, default_value = "examples/rust/cfg.toml")] + #[arg(short = 'C', long, default_value = "examples/cfg.toml")] config: String, } @@ -21,10 +21,14 @@ fn main() { let config_path = Path::new(&args.config); let config = config::Config::builder() .add_source(File::from(config_path)) - .build().unwrap(); + .build() + .unwrap(); // No scope - println!("param1={}\t// No scope", get_param!(example.param1, "default".to_string())); + println!( + "param1={}\t// No scope", + get_param!(example.param1, "default".to_string()) + ); with_params! { // Scope with configuration file parameters params config.param_scope(); diff --git a/core/examples/clap_mini.rs b/core/examples/clap_mini.rs index e685682..f16e55c 100644 --- a/core/examples/clap_mini.rs +++ b/core/examples/clap_mini.rs @@ -2,7 +2,7 @@ use clap::Parser; use hyperparameter::*; #[derive(Parser)] -#[command(after_long_help=generate_params_help())] +#[command(after_long_help=generate_params_help())] // Displays parameter helps when ` --help` is executed. struct CommandLineArgs { /// Specifies hyperparameters in the format `-D key=value` via the command line. #[arg(short = 'D', long)] @@ -13,9 +13,8 @@ fn main() { let args = CommandLineArgs::parse(); with_params! { params ParamScope::from(&args.define); + // Retrieves `example.param1` with a default value of `1` if not specified. - println!("param1={}", get_param!(example.param1, 1)); - // Displays a help message when ` --help` is executed. - println!("param2={}", get_param!(example.param2, false, "help for example.param2")); + println!("param1={}", get_param!(example.param1, false, "help for example.param1")); } -} \ No newline at end of file +} diff --git a/core/src/api.rs b/core/src/api.rs index 0ff0460..76cab3e 100644 --- a/core/src/api.rs +++ b/core/src/api.rs @@ -124,10 +124,10 @@ where key ); if let ParamScope::Just(changes) = self { - if changes.contains_key(&key) { - changes.update(key, val); + if let std::collections::btree_map::Entry::Vacant(e) = changes.entry(key) { + e.insert(Entry::new("", val)); } else { - changes.insert(key, Entry::new("", val)); + changes.update(key, val); } } } @@ -148,11 +148,17 @@ where fn put(&mut self, key: K, val: V) { let hkey = key.xxh(); if let ParamScope::Just(changes) = self { - if changes.contains_key(&hkey) { - changes.update(hkey, val); - } else { + // if changes.contains_key(&hkey) { + // changes.update(hkey, val); + // } else { + // let key: String = key.into(); + // changes.insert(hkey, Entry::new(key, val)); + // } + if let std::collections::btree_map::Entry::Vacant(e) = changes.entry(hkey) { let key: String = key.into(); - changes.insert(hkey, Entry::new(key, val)); + e.insert(Entry::new(key, val)); + } else { + changes.update(hkey, val); } } else { THREAD_STORAGE.with(|ts| ts.borrow_mut().put(key, val)) @@ -167,15 +173,15 @@ pub fn frozen() { #[macro_export] macro_rules! get_param { ($name:expr, $default:expr) => {{ - const CONST_KEY: &str = const_str::replace!(stringify!($name), ";", ""); - const CONST_HASH: u64 = xxhash_rust::const_xxh64::xxh64(CONST_KEY.as_bytes(), 42); + const CONST_KEY: &str = ::const_str::replace!(stringify!($name), ";", ""); + const CONST_HASH: u64 = ::xxhash_rust::const_xxh64::xxh64(CONST_KEY.as_bytes(), 42); THREAD_STORAGE.with(|ts| ts.borrow_mut().get_or_else(CONST_HASH, $default)) // ParamScope::default().get_or_else(CONST_HASH, $default) }}; ($name:expr, $default:expr, $help: expr) => {{ - const CONST_KEY: &str = const_str::replace!(stringify!($name), ";", ""); - const CONST_HASH: u64 = xxhash_rust::const_xxh64::xxh64(CONST_KEY.as_bytes(), 42); + const CONST_KEY: &str = ::const_str::replace!(stringify!($name), ";", ""); + const CONST_HASH: u64 = ::xxhash_rust::const_xxh64::xxh64(CONST_KEY.as_bytes(), 42); // ParamScope::default().get_or_else(CONST_HASH, $default) { const CONST_HELP: &str = $help; @@ -221,7 +227,7 @@ macro_rules! with_params { ) =>{ let mut ps = ParamScope::default(); { - const CONST_KEY: &str = const_str::replace!(stringify!($($key).+), ";", ""); + const CONST_KEY: &str = ::const_str::replace!(stringify!($($key).+), ";", ""); ps.put(CONST_KEY, $val); } with_params!(params ps; $($body)*) @@ -234,7 +240,7 @@ macro_rules! with_params { $($body:tt)* ) => { { - const CONST_KEY: &str = const_str::replace!(stringify!($($key).+), ";", ""); + const CONST_KEY: &str = ::const_str::replace!(stringify!($($key).+), ";", ""); $ps.put(CONST_KEY, $val); } with_params!(params $ps; $($body)*) @@ -322,7 +328,7 @@ macro_rules! with_params_readonly { ) =>{ let mut ps = ParamScope::default(); { - const CONST_KEY: &str = const_str::replace!(stringify!($($key).+), ";", ""); + const CONST_KEY: &str = ::const_str::replace!(stringify!($($key).+), ";", ""); ps.put(CONST_KEY, $val); } with_params!(params ps; $($body)*) @@ -491,7 +497,7 @@ mod tests { #[test] fn test_param_scope_with_param_mixed_get_set() { with_params! { - get a_b_c = a.b.c or 1; + get _a_b_c = a.b.c or 1; set a.b.c = 3; get a_b_c = a.b.c or 2; diff --git a/core/src/cfg.rs b/core/src/cfg.rs index 686055e..8d9c66e 100644 --- a/core/src/cfg.rs +++ b/core/src/cfg.rs @@ -6,16 +6,14 @@ pub use crate::storage::THREAD_STORAGE; pub use crate::value::Value; pub use crate::xxh::XXHashable; -use config; - pub trait AsParamScope { - fn param_scope(self: &Self) -> ParamScope; + fn param_scope(&self) -> ParamScope; } impl AsParamScope for config::Config { - fn param_scope(self: &Self) -> ParamScope { + fn param_scope(&self) -> ParamScope { let mut ps = ParamScope::default(); - fn unpack(ps: &mut ParamScope, prefix: Option, value: config::Value) -> () { + fn unpack(ps: &mut ParamScope, prefix: Option, value: config::Value) { match (prefix, value.kind) { (None, config::ValueKind::Table(v)) => v.iter().for_each(|(k, v)| { unpack(ps, Some(k.to_string()), v.clone()); @@ -25,7 +23,7 @@ impl AsParamScope for config::Config { (Some(k), config::ValueKind::Float(v)) => ps.put(k, v), (Some(k), config::ValueKind::String(v)) => ps.put(k, v), (Some(prefix), config::ValueKind::Table(v)) => v.iter().for_each(|(k, v)| { - unpack(ps, Some(format!("{}.{}", prefix, k.to_string())), v.clone()); + unpack(ps, Some(format!("{}.{}", prefix, k)), v.clone()); }), _ => todo!(), }; diff --git a/core/src/cli.rs b/core/src/cli.rs index ee3ea1c..e6b06bb 100644 --- a/core/src/cli.rs +++ b/core/src/cli.rs @@ -21,7 +21,7 @@ pub fn generate_params_help() -> String { let mut params: Vec<_> = params .iter() .map(|kv| { - let mut descs = Vec::from_iter(kv.1.iter().map(|x| x.clone())); + let mut descs = Vec::from_iter(kv.1.iter().cloned()); descs.sort(); (kv.0.clone(), descs.join("\n\t")) }) diff --git a/core/src/lib.rs b/core/src/lib.rs index aa8f821..5eb0f3b 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -2,28 +2,23 @@ #[macro_use] extern crate proptest; -pub extern crate const_str; -pub extern crate xxhash_rust; +mod storage; +mod value; + +mod api; +mod cfg; +mod cli; +mod ffi; +mod xxh; pub use crate::api::frozen; pub use crate::api::ParamScope; pub use crate::api::ParamScopeOps; pub use crate::cfg::AsParamScope; +pub use crate::cli::generate_params_help; +pub use crate::cli::PARAMS; pub use crate::storage::GetOrElse; pub use crate::storage::THREAD_STORAGE; pub use crate::value::Value; +pub use crate::xxh::xxhash; pub use crate::xxh::XXHashable; - -pub mod storage; -pub mod value; - -pub mod api; -pub mod cfg; -pub mod ffi; -pub mod xxh; - -// #[cfg(clap)] -pub mod cli; -// #[cfg(clap)] -pub use crate::cli::PARAMS; -pub use crate::cli::generate_params_help; \ No newline at end of file diff --git a/core/src/storage.rs b/core/src/storage.rs index 8bad3a2..84de864 100644 --- a/core/src/storage.rs +++ b/core/src/storage.rs @@ -68,11 +68,6 @@ impl MultipleVersion for Tree { } } -// pub fn hashstr>(s: T) -> u64 { -// let s: String = s.into(); -// xxhstr(&s) -// } - thread_local! { pub static THREAD_STORAGE: RefCell = create_thread_storage(); } @@ -157,16 +152,13 @@ impl Storage { if self.history.last().unwrap().contains(&hkey) { self.tree.update(hkey, val); } else { - if self.tree.contains_key(&hkey) { - self.tree.revision(hkey, val); + if let std::collections::btree_map::Entry::Vacant(e) = self.tree.entry(hkey) { + e.insert(Entry { + key, + val: VersionedValue::from(val.into()), + }); } else { - self.tree.insert( - hkey, - Entry { - key, - val: VersionedValue::from(val.into()), - }, - ); + self.tree.revision(hkey, val); } self.history.last_mut().unwrap().insert(hkey); } @@ -182,18 +174,10 @@ impl Storage { } } - // pub fn rollback>(&mut self, key: T) { - // let hkey = hashstr(key); - // self.tree.rollback(hkey); - // } - pub fn keys(&self) -> Vec { self.tree .values() - .filter(|x| match x.value() { - Value::Empty => false, - _ => true, - }) + .filter(|x| !matches!(x.value(), Value::Empty)) .map(|x| x.key.clone()) .collect() } diff --git a/core/src/value.rs b/core/src/value.rs index 176a6ed..540a57c 100644 --- a/core/src/value.rs +++ b/core/src/value.rs @@ -17,7 +17,7 @@ pub type DeferSafe = Arc; /// The value type for hyperparameter values /// /// ``` -/// use hyperparameter::value::Value; +/// use hyperparameter::Value; /// let v: Value = 1i32.into(); /// println!("{:?}", v); /// ``` @@ -282,7 +282,7 @@ impl VersionedValue { pub fn rollback(&mut self) -> bool { self.0.pop_front(); - self.0.len() > 0 + !self.0.is_empty() } } @@ -360,7 +360,7 @@ mod test_versioned_value { #[test] fn test_versioned_value() { - let mut val = VersionedValue::from::(0i64.into()); + let mut val = VersionedValue::from::(0i64); assert_eq!(format!("{:?}", val), "VersionedValue([Int(0)])"); val.update(2.0); diff --git a/hyperparameter/src/ext.rs b/hyperparameter/src/ext.rs index b180b69..99e5ff5 100644 --- a/hyperparameter/src/ext.rs +++ b/hyperparameter/src/ext.rs @@ -67,7 +67,7 @@ impl KVStorage { if k == UserDefinedType::PyObjectType as i32 { res.set_item(k, PyAny::from_owned_ptr(py, v as *mut pyo3::ffi::PyObject)) } else { - res.set_item(k, v as u64) + res.set_item(k, v) } } }