Refactoring Makefiles & clean up tests

tests/common/Makefile

@@ -1,7 +1,8 @@
-# this is dupped from vars.mk
-ROOT:= $(shell git rev-parse --show-toplevel)
-RS_SOURCES := $(wildcard $(SRC)/*.rs)
-C_LIB_PATH := $(ROOT)/target/debug/libhyperbee.so
+# This file is loaded from other Makefile's so we can't use simple "include ../foo" because 
+# that is resolved relative to the original Makefile.
+#
+# $(dir $(lastword $(MAKEFILE_LIST))) gets the dir of the current Makefile
+include $(dir $(lastword $(MAKEFILE_LIST)))vars.mk
 
 # Build Rust FFI library
 $(C_LIB_PATH): $(RS_SOURCES) $(ROOT)/Cargo.toml

tests/common/js/mod.rs

@@ -1,4 +1,4 @@
-use super::{git_root, join_paths, run_code, run_make_from_with};
+use super::{build_whole_script, git_root, join_paths, run_code, run_make_from_dir_with_arg};
 use std::{
     path::{Path, PathBuf},
     process::Output,
@@ -8,7 +8,7 @@ pub static REL_PATH_TO_NODE_MODULES: &str = "./tests/common/js/node_modules";
 pub static REL_PATH_TO_JS_DIR: &str = "./tests/common/js";
 
 pub fn require_js_data() -> Result<(), Box<dyn std::error::Error>> {
-    let _ = run_make_from_with(REL_PATH_TO_JS_DIR, "")?;
+    let _ = run_make_from_dir_with_arg(REL_PATH_TO_JS_DIR, "")?;
     Ok(())
 }
 
@@ -56,26 +56,18 @@ fn no_write_pre_script(storage_dir: &str) -> String {
 }
 
 pub fn run_js(storage_dir: &str, script: &str) -> Result<Output, Box<dyn std::error::Error>> {
-    run_code(
-        storage_dir,
-        no_write_pre_script,
-        script,
-        POST_SCRIPT,
-        SCRIPT_FILE_NAME,
-        build_command,
-        vec![],
-    )
+    require_js_data()?;
+    let code = build_whole_script(&no_write_pre_script(storage_dir), script, POST_SCRIPT);
+    run_code(&code, SCRIPT_FILE_NAME, build_command, vec![])
 }
 
 pub fn run_js_writable<T: AsRef<Path>>(storage_dir: T, script: &str) -> super::Result<Output> {
+    require_js_data()?;
     let path = storage_dir.as_ref();
-    run_code(
-        &path.to_string_lossy(),
-        writable_pre_script,
+    let code = build_whole_script(
+        &writable_pre_script(&path.to_string_lossy()),
         script,
         POST_SCRIPT,
-        SCRIPT_FILE_NAME,
-        build_command,
-        vec![],
-    )
+    );
+    run_code(&code, SCRIPT_FILE_NAME, build_command, vec![])
 }

tests/common/mod.rs

@@ -6,6 +6,7 @@
 use std::{
     fs::File,
     io::Write,
+    path::PathBuf,
     process::{Command, Output},
     sync::atomic::{AtomicU64, Ordering},
 };
@@ -51,8 +52,8 @@ pub fn git_root() -> Result<String> {
     Ok(String::from_utf8(x.stdout)?.trim().to_string())
 }
 
-pub fn path_to_c_lib() -> Result<String> {
-    Ok(join_paths!(git_root()?, PATH_TO_C_LIB))
+pub fn path_to_c_lib() -> Result<PathBuf> {
+    Ok(join_paths!(git_root()?, PATH_TO_C_LIB).parse()?)
 }
 
 pub fn get_data_dir() -> Result<String> {
@@ -66,31 +67,30 @@ pub fn run_script_relative_to_git_root(script: &str) -> Result<Output> {
         .output()?)
 }
 
+pub fn build_whole_script(pre_code: &str, code: &str, post_code: &str) -> String {
+    format!(
+        "{pre_code}
+{code}
+{post_code}
+"
+    )
+}
+
+/// Run some code.
+/// Put the provided `code` into a file named `script_file_name` inside a temporary directory where
+/// `copy_dirs` are copied into. `build_command` should output a shell command as a string that
+/// runs the script.
 pub fn run_code(
-    storage_dir: &str,
-    build_pre_script: impl FnOnce(&str) -> String,
-    script: &str,
-    post_script: &str,
+    code: &str,
     script_file_name: &str,
     build_command: impl FnOnce(&str, &str) -> String,
-    copy_dirs: Vec<String>,
+    copy_dirs: Vec<PathBuf>,
 ) -> Result<Output> {
-    let storage_dir_name = storage_dir.to_string();
-    let pre_script = build_pre_script(&storage_dir_name);
-
     let working_dir = tempfile::tempdir()?;
-
-    let code = format!(
-        "{pre_script}
-{script}
-{post_script}
-"
-    );
+    let working_dir_path = working_dir.path().display().to_string();
     let script_path = working_dir.path().join(script_file_name);
-    let script_file = File::create(&script_path)?;
-    write!(&script_file, "{}", &code)?;
+    write!(&File::create(&script_path)?, "{code}")?;
 
-    let working_dir_path = working_dir.path().display().to_string();
     // copy dirs into working dir
     for dir in copy_dirs {
         let dir_cp_cmd = Command::new("cp")
@@ -100,17 +100,17 @@ pub fn run_code(
             .output()?;
         if dir_cp_cmd.status.code() != Some(0) {
             return Err(Box::new(Error::TestError(format!(
-                "failed to copy dir [{dir}] to [{working_dir_path}] got stderr: {}",
+                "failed to copy dir [{}] to [{working_dir_path}] got stderr: {}",
+                dir.display(),
                 String::from_utf8_lossy(&dir_cp_cmd.stderr),
             ))));
         }
     }
-    let script_path_str = script_path.display().to_string();
-    let cmd = build_command(&working_dir_path, &script_path_str);
+    let cmd = build_command(&working_dir_path, &script_path.to_string_lossy());
     check_cmd_output(Command::new("sh").arg("-c").arg(cmd).output()?)
 }
 
-pub fn run_make_from_with(dir: &str, arg: &str) -> Result<Output> {
+pub fn run_make_from_dir_with_arg(dir: &str, arg: &str) -> Result<Output> {
     let path = join_paths!(git_root()?, dir);
     let cmd = format!("cd {path} && flock make.lock make {arg} && rm -f make.lock ");
     let out = check_cmd_output(Command::new("sh").arg("-c").arg(cmd).output()?)?;

tests/common/python/Makefile

@@ -1,13 +1,8 @@
 include ../vars.mk
 
+# Build the Python Hyperbee library
+$(TARGET_DEBUG_DIR)/hyperbee.py: $(C_LIB_PATH)
+	cargo run -F ffi --bin uniffi-bindgen generate --library $(C_LIB_PATH) --language python --out-dir $(TARGET_DEBUG_DIR)
 
-target/hyperbee.py: $(C_LIB_PATH) target/libhyperbee.so | target
-	cargo run -F ffi --bin uniffi-bindgen generate --library $(C_LIB_PATH) --language python --out-dir target
-
-target/libhyperbee.so: $(C_LIB_PATH) | target
-	cp $(C_LIB_PATH) target/.
-
-target:
-	mkdir -p $@
-
+# Pull in the $(C_LIB_PATH) rule
 include ../Makefile

tests/common/python/mod.rs

@@ -2,12 +2,14 @@ use std::{path::PathBuf, process::Output};
 
 use tempfile::tempdir;
 
-use super::{git_root, join_paths, path_to_c_lib, run_code, run_make_from_with};
+use super::{
+    build_whole_script, git_root, join_paths, path_to_c_lib, run_code, run_make_from_dir_with_arg,
+};
 
 static REL_PATH_TO_HERE: &str = "./tests/common/python";
 static PRE_SCRIPT: &str = "
 import asyncio
-from target.hyperbee import *
+from hyperbee import *
 ";
 
 static POST_SCRIPT: &str = "
@@ -20,25 +22,22 @@ fn build_command(working_dir: &str, script_path: &str) -> String {
 }
 
 pub fn path_to_python_target() -> Result<PathBuf, Box<dyn std::error::Error>> {
-    let p = join_paths!(&git_root()?, &REL_PATH_TO_HERE, "target");
+    let p = join_paths!(&git_root()?, "target/debug/hyperbee.py");
     Ok(p.into())
 }
 
-pub fn run_python(script: &str) -> Result<Output, Box<dyn std::error::Error>> {
-    let storage_dir = tempdir()?;
-    let target_path = path_to_python_target()?.display().to_string();
-    let storage_dir_path = format!("{}", storage_dir.path().display());
+/// Run the provided python `code` within a context where the Python Hyperbee library is imported.
+/// Code must be written within a `async main(): ...` function.
+pub fn run_python(code: &str) -> Result<Output, Box<dyn std::error::Error>> {
+    require_python()?;
     run_code(
-        &storage_dir_path,
-        |_| PRE_SCRIPT.to_string(),
-        script,
-        POST_SCRIPT,
+        &build_whole_script(PRE_SCRIPT, code, POST_SCRIPT),
         "script.py",
         build_command,
-        vec![target_path, path_to_c_lib()?],
+        vec![path_to_python_target()?, path_to_c_lib()?],
     )
 }
 
 pub fn require_python() -> Result<Output, Box<dyn std::error::Error>> {
-    run_make_from_with(REL_PATH_TO_HERE, "")
+    run_make_from_dir_with_arg(REL_PATH_TO_HERE, "")
 }

tests/common/vars.mk

@@ -2,4 +2,5 @@ ROOT:= $(shell git rev-parse --show-toplevel)
 SRC := $(ROOT)/src
 C_SOURCES := $(wildcard *.c);
 RS_SOURCES := $(wildcard $(SRC)/*.rs)
-C_LIB_PATH := $(ROOT)/target/debug/libhyperbee.so
+TARGET_DEBUG_DIR := $(ROOT)/target/debug
+C_LIB_PATH := $(TARGET_DEBUG_DIR)/libhyperbee.so

tests/py_tests.rs

@@ -0,0 +1,141 @@
+mod common;
+use common::{parse_json_result, python::run_python, write_range_to_hb, Result};
+use hyperbee::Hyperbee;
+
+#[tokio::test]
+async fn hello_world() -> Result<()> {
+    let storage_dir = tempfile::tempdir()?;
+    {
+        let hb = Hyperbee::from_storage_dir(&storage_dir).await?;
+        hb.put(b"hello", Some(b"world")).await?;
+    }
+    let result = run_python(&format!(
+        "
+async def main():
+    hb = await hyperbee_from_storage_dir('{}')
+    res = await hb.get(b'hello')
+    assert(res.value == b'world')
+",
+        storage_dir.path().display()
+    ))?;
+
+    assert_eq!(result.status.code(), Some(0));
+    Ok(())
+}
+#[tokio::test]
+async fn test_sub_prefix() -> Result<()> {
+    let out = run_python(
+        "
+async def main():
+    prefix = b'myprefix'
+    key = b'keykey'
+    val_no_pref = b'no prefix'
+    val_with_pref = b'yes prefix'
+
+    config = PrefixedConfig(bytes([0]))
+    hb = await hyperbee_from_ram()
+    sub_hb = await hb.sub(prefix, config)
+
+    await hb.put(key, val_no_pref)
+    await sub_hb.put(key, val_with_pref)
+
+    x = await hb.get(key)
+    assert(x.value == val_no_pref)
+
+    x = await sub_hb.get(key)
+    assert(x.value == val_with_pref)
+",
+    )?;
+    assert_eq!(out.status.code(), Some(0));
+    Ok(())
+}
+
+#[tokio::test]
+async fn get_set_del() -> Result<()> {
+    let out = run_python(
+        "
+async def main():
+    hb = await hyperbee_from_ram()
+    x = await hb.put(b'hello', b'world')
+    assert(x.old_seq == None)
+    assert(x.new_seq == 1)
+
+    x = await hb.get(b'hello')
+    assert(x.value == b'world')
+
+    x = await hb.delete(b'hello')
+    assert(x == 1)
+",
+    )?;
+    assert_eq!(out.status.code(), Some(0));
+    Ok(())
+}
+
+#[tokio::test]
+async fn optionals() -> Result<()> {
+    let storage_dir = tempfile::tempdir()?;
+    {
+        let hb = Hyperbee::from_storage_dir(&storage_dir).await?;
+        let _ = hb.put(b"hello", None).await?;
+    }
+    let code = format!(
+        "
+async def main():
+    import json
+    hb = await hyperbee_from_storage_dir('{}')
+    res = await hb.get(b'hello')
+    assert(res.value is None)
+
+    res = await hb.put(b'skipval', None)
+    res = await hb.get(b'skipval')
+    assert(res.value is None)
+",
+        storage_dir.path().display()
+    );
+
+    let output = run_python(&code)?;
+    assert_eq!(output.status.code(), Some(0));
+    Ok(())
+}
+
+#[tokio::test]
+async fn check_version() -> Result<()> {
+    let out = run_python(
+        "
+async def main():
+    hb = await hyperbee_from_ram()
+    x = await hb.version()
+    assert(x == 0)
+    await hb.put(b'foo', None)
+    x = await hb.version()
+    assert(x == 2)
+",
+    )?;
+    assert_eq!(out.status.code(), Some(0));
+    Ok(())
+}
+
+#[tokio::test]
+async fn zero_to_one_hundred() -> Result<()> {
+    let storage_dir = tempfile::tempdir()?;
+    let hb = Hyperbee::from_storage_dir(&storage_dir).await?;
+    let keys = write_range_to_hb!(&hb);
+    let code = format!(
+        "
+async def main():
+    import json
+    hb = await hyperbee_from_storage_dir('{}')
+    keys = [bytes(str(i), 'utf8') for i in range(100)]
+    results = [await hb.get(k) for k in keys]
+    values = [res.value.decode('utf8') for res in results]
+    print(json.dumps(values))
+",
+        storage_dir.path().display()
+    );
+
+    let output = run_python(&code)?;
+    let res = parse_json_result(&output)?;
+    assert_eq!(res, keys);
+    assert_eq!(output.status.code(), Some(0));
+    Ok(())
+}