From d78e9da0760923f87755acdc3a2058e6dfbc9d7a Mon Sep 17 00:00:00 2001 From: meo Date: Mon, 6 Jan 2025 12:18:33 +0100 Subject: [PATCH 1/5] add(ignore): ignore of .vscode file add the vscode config file to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a00bc94..629c291 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ test.log error.log rebalance_db.txt -testing_data \ No newline at end of file +testing_data +.vscode \ No newline at end of file From 10a5a91302067e2295b8e2e7e2aa6fdccb73cbda Mon Sep 17 00:00:00 2001 From: meo Date: Mon, 6 Jan 2025 13:14:07 +0100 Subject: [PATCH 2/5] add(test): add set of test - test 1000 1Ko file - test 5 1 Go file - test whit both --- testing.sh | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/testing.sh b/testing.sh index 4fe7333..45c5771 100755 --- a/testing.sh +++ b/testing.sh @@ -9,6 +9,7 @@ log_std_file=./test.log log_error_file=./error.log test_data_src=./test/pool test_pool_data_path=./testing_data +test_pool_data_size_path=$test_pool_data_path/size ## Color Constants @@ -18,6 +19,7 @@ Color_Off='\033[0m' # Text Reset # Regular Colors Red='\033[0;31m' # Red Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow Cyan='\033[0;36m' # Cyan ## Functions @@ -63,6 +65,14 @@ function assert_matching_file_not_copied() { fi } +function print_time_taken(){ + time_taken=$1 + minute=$((time_taken / 60000)) + seconde=$((time_taken % 60000 / 1000)) + miliseconde=$((time_taken % 1000)) + color_echo "$Yellow" "Time taken: ${minute}m ${seconde}s ${miliseconde}ms" +} + color_echo "$Cyan" "Running tests..." color_echo "$Cyan" "Running tests with default options..." @@ -108,4 +118,51 @@ assert_matching_file_not_copied "mp4.txt" assertions color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with different file count and size..." +prepare + +mkdir -p $test_pool_data_size_path + +color_echo "$Cyan" "Creating 1000 files of 1KB each..." +mkdir -p $test_pool_data_size_path/small +for i in {1..1000}; do + dd if=/dev/urandom of=$test_pool_data_size_path/small/file$i.txt bs=1024 count=1 >> /dev/null 2>&1 +done + +color_echo "$Cyan" "Creating 5 file of 1GB each..." +mkdir -p $test_pool_data_size_path/big +for i in {1..5}; do + dd if=/dev/urandom of=$test_pool_data_size_path/big/file$i.txt bs=1024 count=1048576 >> /dev/null 2>&1 +done + +color_echo "$Green" "Files created!" + +echo "Running rebalancing on small files..." +# measure time taken +start_time=$(date +%s%3N) +./zfs-inplace-rebalancing.sh $test_pool_data_size_path/small >> $log_std_file 2>> $log_error_file +end_time=$(date +%s%3N) +print_time_taken $((end_time - start_time)) +assertions +color_echo "$Green" "Tests passed!" + +echo "Running rebalancing on big files..." +# measure time taken +start_time=$(date +%s%3N) +./zfs-inplace-rebalancing.sh $test_pool_data_size_path/big >> $log_std_file 2>> $log_error_file +end_time=$(date +%s%3N) +print_time_taken $((end_time - start_time)) +assertions +color_echo "$Green" "Tests passed!" + +echo "Running rebalancing on all files..." +# measure time taken whit +start_time=$(date +%s%3N) +./zfs-inplace-rebalancing.sh $test_pool_data_size_path >> $log_std_file 2>> $log_error_file +end_time=$(date +%s%3N) +print_time_taken $((end_time - start_time)) +assertions +color_echo "$Green" "Tests passed!" + + color_echo "$Green" "All tests passed!" \ No newline at end of file From a9613f046fe22ffd8596d68055927315e8f6a815 Mon Sep 17 00:00:00 2001 From: meo Date: Mon, 6 Jan 2025 13:24:03 +0100 Subject: [PATCH 3/5] fix: spellcheck prevent wordspliting --- testing.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing.sh b/testing.sh index 45c5771..e25bd6c 100755 --- a/testing.sh +++ b/testing.sh @@ -126,13 +126,13 @@ mkdir -p $test_pool_data_size_path color_echo "$Cyan" "Creating 1000 files of 1KB each..." mkdir -p $test_pool_data_size_path/small for i in {1..1000}; do - dd if=/dev/urandom of=$test_pool_data_size_path/small/file$i.txt bs=1024 count=1 >> /dev/null 2>&1 + dd if=/dev/urandom of=$test_pool_data_size_path/small/file_"$i".txt bs=1024 count=1 >> /dev/null 2>&1 done color_echo "$Cyan" "Creating 5 file of 1GB each..." mkdir -p $test_pool_data_size_path/big for i in {1..5}; do - dd if=/dev/urandom of=$test_pool_data_size_path/big/file$i.txt bs=1024 count=1048576 >> /dev/null 2>&1 + dd if=/dev/urandom of=$test_pool_data_size_path/big/file_"$i".txt bs=1024 count=1048576 >> /dev/null 2>&1 done color_echo "$Green" "Files created!" From 69a087dcfaf65b0049d8247665da425228d6d58b Mon Sep 17 00:00:00 2001 From: meo Date: Tue, 7 Jan 2025 12:12:20 +0100 Subject: [PATCH 4/5] fix: remove of db remove the db before testing for more accurate time --- testing.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/testing.sh b/testing.sh index e25bd6c..108e440 100755 --- a/testing.sh +++ b/testing.sh @@ -147,6 +147,7 @@ assertions color_echo "$Green" "Tests passed!" echo "Running rebalancing on big files..." +rm -f rebalance_db.txt # measure time taken start_time=$(date +%s%3N) ./zfs-inplace-rebalancing.sh $test_pool_data_size_path/big >> $log_std_file 2>> $log_error_file @@ -156,7 +157,8 @@ assertions color_echo "$Green" "Tests passed!" echo "Running rebalancing on all files..." -# measure time taken whit +rm -f rebalance_db.txt +# measure time taken start_time=$(date +%s%3N) ./zfs-inplace-rebalancing.sh $test_pool_data_size_path >> $log_std_file 2>> $log_error_file end_time=$(date +%s%3N) @@ -164,5 +166,10 @@ print_time_taken $((end_time - start_time)) assertions color_echo "$Green" "Tests passed!" +color_echo "$Green" "All tests passed!" +color_echo "$Cyan" "Cleaning" +rm -f $log_std_file +rm -f $log_error_file +rm -f rebalance_db.txt +rm -rf $test_pool_data_path -color_echo "$Green" "All tests passed!" \ No newline at end of file From 5b194a22f7e1898d11dacfe056c1fdc60b3868e6 Mon Sep 17 00:00:00 2001 From: meo Date: Tue, 7 Jan 2025 14:50:06 +0100 Subject: [PATCH 5/5] fix: time in macOS and freeBSD - use of gdate (coreutils) for macOS - switch to `ns` in place of `ms` for freeBSD --- .github/workflows/test.yml | 3 +++ testing.sh | 36 +++++++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 02e3b01..b014837 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install coreutils + run: brew install coreutils + - name: Run testing script on macOS run: ./testing.sh diff --git a/testing.sh b/testing.sh index 108e440..f531e1e 100755 --- a/testing.sh +++ b/testing.sh @@ -42,6 +42,21 @@ function prepare() { cp -rf $test_data_src $test_pool_data_path } +# return time to the milisecond +function get_time() { + + case "$OSTYPE" in + darwin*) + date=$(gdate +%s%N) + ;; + *) + date=$(date +%s%N) + ;; + esac + + echo "$date" +} + function assertions() { # check error log is empty if grep -q '[^[:space:]]' $log_error_file; then @@ -139,30 +154,33 @@ color_echo "$Green" "Files created!" echo "Running rebalancing on small files..." # measure time taken -start_time=$(date +%s%3N) +start_time=$(get_time) ./zfs-inplace-rebalancing.sh $test_pool_data_size_path/small >> $log_std_file 2>> $log_error_file -end_time=$(date +%s%3N) -print_time_taken $((end_time - start_time)) +end_time=$(get_time) +time_taken=$(( (end_time - start_time) / 1000000 )) +print_time_taken $time_taken assertions color_echo "$Green" "Tests passed!" echo "Running rebalancing on big files..." rm -f rebalance_db.txt # measure time taken -start_time=$(date +%s%3N) +start_time=$(get_time) ./zfs-inplace-rebalancing.sh $test_pool_data_size_path/big >> $log_std_file 2>> $log_error_file -end_time=$(date +%s%3N) -print_time_taken $((end_time - start_time)) +end_time=$(get_time) +time_taken=$(( (end_time - start_time) / 1000000 )) +print_time_taken $time_taken assertions color_echo "$Green" "Tests passed!" echo "Running rebalancing on all files..." rm -f rebalance_db.txt # measure time taken -start_time=$(date +%s%3N) +start_time=$(get_time) ./zfs-inplace-rebalancing.sh $test_pool_data_size_path >> $log_std_file 2>> $log_error_file -end_time=$(date +%s%3N) -print_time_taken $((end_time - start_time)) +end_time=$(get_time) +time_taken=$(( (end_time - start_time) / 1000000 )) +print_time_taken $time_taken assertions color_echo "$Green" "Tests passed!"