diff --git a/Makefile b/Makefile index 66f820419..4f75bcb88 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ -.PHONY: compile +.PHONY: compile test compile: rebar3 compile +test: + scripts/parallel_tests.sh + WAMR_VERSION = 2.2.0 WAMR_DIR = _build/wamr diff --git a/rebar.config b/rebar.config index ddf8cea29..2adc75a0b 100644 --- a/rebar.config +++ b/rebar.config @@ -56,6 +56,7 @@ {d, 'ENABLE_ROCKSDB', true} ]} ]}, + {test, [{erl_opts, [{d, 'DEFAULT_HTTP_CLIENT', hb_test_cache}]}]}, {http3, [ {deps, [ {quicer, {git, "https://github.com/emqx/quic.git", diff --git a/scripts/parallel_tests.sh b/scripts/parallel_tests.sh new file mode 100755 index 000000000..792e1ee07 --- /dev/null +++ b/scripts/parallel_tests.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# +# Run EUnit test modules in parallel across separate BEAM VMs. +# Each VM gets a unique HB_PORT to avoid port conflicts. +# +# Usage: +# scripts/parallel_tests.sh # run all test modules +# scripts/parallel_tests.sh mod1 mod2 mod3 # run specific modules +# +set -uo pipefail + +NPROC=$(nproc 2>/dev/null || echo 16) +BASE_PORT=19000 +EBIN_DIR="_build/test/lib/hb/ebin" +PA_DIRS="_build/test/lib/*/ebin" +LOG_DIR="/tmp/eunit_parallel" +mkdir -p "$LOG_DIR" + +echo "=== Compiling (test profile) ===" +rebar3 as test compile || { echo "Compilation failed"; exit 1; } + +if [ $# -gt 0 ]; then + MODULES="$*" +else + echo "=== Discovering test modules ===" + MODULES=$(erl -noshell -pa $PA_DIRS -eval ' + Beams = filelib:wildcard("'"$EBIN_DIR"'/*.beam"), + Mods = [list_to_atom(filename:basename(B, ".beam")) || B <- Beams], + TestMods = lists:filter(fun(M) -> + try + Exports = M:module_info(exports), + lists:any(fun({F,0}) -> + S = atom_to_list(F), + lists:suffix("_test", S) orelse lists:suffix("_test_", S); + (_) -> false end, Exports) + catch _:_ -> false + end + end, Mods), + [io:format("~s~n", [M]) || M <- lists:sort(TestMods)], + halt(0). + ' 2>/dev/null) +fi + +MODULE_COUNT=$(echo "$MODULES" | wc -w) +echo "=== Running $MODULE_COUNT modules with $NPROC workers ===" +echo "" + +T_START=$(date +%s) + +run_module() { + local mod=$1 + local port=$2 + local logfile="$LOG_DIR/${mod}.log" + + local t0 + t0=$(date +%s%3N 2>/dev/null || python3 -c 'import time; print(int(time.time()*1000))') + + HB_PORT=$port erl -noshell -pa $PA_DIRS -eval " + application:ensure_all_started(hb), + case eunit:test($mod, [verbose]) of + ok -> halt(0); + error -> halt(1) + end. + " > "$logfile" 2>&1 + local rc=$? + + local t1 + t1=$(date +%s%3N 2>/dev/null || python3 -c 'import time; print(int(time.time()*1000))') + local elapsed=$(( (t1 - t0) / 1000 )) + + if [ $rc -eq 0 ]; then + printf " \033[32mPASS\033[0m %-45s %3ds\n" "$mod" "$elapsed" + else + printf " \033[31mFAIL\033[0m %-45s %3ds (see %s)\n" "$mod" "$elapsed" "$logfile" + fi + return $rc +} + +export -f run_module +export PA_DIRS LOG_DIR + +FAILED=0 +PORT=$BASE_PORT +PIDS=() +MODS_ARRAY=() + +for mod in $MODULES; do + run_module "$mod" "$PORT" & + PIDS+=($!) + MODS_ARRAY+=("$mod") + PORT=$((PORT + 1)) + + # Throttle to NPROC concurrent jobs + if [ ${#PIDS[@]} -ge "$NPROC" ]; then + wait "${PIDS[0]}" || FAILED=$((FAILED + 1)) + PIDS=("${PIDS[@]:1}") + MODS_ARRAY=("${MODS_ARRAY[@]:1}") + fi +done + +for pid in "${PIDS[@]}"; do + wait "$pid" || FAILED=$((FAILED + 1)) +done + +T_END=$(date +%s) +ELAPSED=$((T_END - T_START)) + +echo "" +echo "=== Done in ${ELAPSED}s ===" +echo " Modules: $MODULE_COUNT" +echo " Failed: $FAILED" + +if [ "$FAILED" -gt 0 ]; then + echo "" + echo "Failed module logs in $LOG_DIR/" + exit 1 +fi diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl index ec21ac443..05e5c0d85 100644 --- a/src/dev_arweave_offset.erl +++ b/src/dev_arweave_offset.erl @@ -353,39 +353,27 @@ parse_offset_test() -> offset_item_cases_test() -> Opts = #{}, - % A simple message. - assert_offset_item( - <<"160399272861859">>, - 498852, - #{ <<"content-type">> => <<"image/png">> }, - Opts - ), - % A reference with a given length. - assert_offset_item( - <<"160399272861859-498852">>, - 498852, - #{ <<"content-type">> => <<"image/png">> }, - Opts - ), - % A reference to a byte in the middle of the test message. - assert_offset_item( - <<"160399273000000">>, - 498852, - #{ <<"content-type">> => <<"image/png">> }, - Opts - ), - % A megabyte reference to the item, occurring in the middle of the item. - assert_offset_item( - <<"160399273m">>, - 498852, - #{ <<"content-type">> => <<"image/png">> }, - Opts - ), - assert_offset_item( - <<"384600234780716">>, - 856691, - #{ <<"content-type">> => <<"image/jpeg">> }, - Opts + hb_pmap:parallel_map( + [ + % A simple message. + {<<"160399272861859">>, 498852, + #{<<"content-type">> => <<"image/png">>}}, + % A reference with a given length. + {<<"160399272861859-498852">>, 498852, + #{<<"content-type">> => <<"image/png">>}}, + % A reference to a byte in the middle of the test message. + {<<"160399273000000">>, 498852, + #{<<"content-type">> => <<"image/png">>}}, + % A megabyte reference to the item, occurring in the middle of the item. + {<<"160399273m">>, 498852, + #{<<"content-type">> => <<"image/png">>}}, + {<<"384600234780716">>, 856691, + #{<<"content-type">> => <<"image/jpeg">>}} + ], + fun({Path, DataSize, Tags}) -> + assert_offset_item(Path, DataSize, Tags, Opts) + end, + 5 ), ok. diff --git a/src/dev_copycat_arweave.erl b/src/dev_copycat_arweave.erl index 970b3d9b4..f1dd3d917 100644 --- a/src/dev_copycat_arweave.erl +++ b/src/dev_copycat_arweave.erl @@ -480,21 +480,17 @@ index_ids_test() -> <<"WbRAQbeyjPHgopBKyi0PLeKWvYZr3rgZvQ7QY3ASJS4">> ) ), - assert_item_read( - <<"0vy2Ey8bWkSDcRIvWQJjxDeVGYOrTSmYIIhBILJntY8">>, - Opts), - assert_item_read( - <<"2lmrYydmDweX2MgGH39ZEB9hKm2JqGOYmRiG3n_xh8A">>, - Opts), - assert_item_read( - <<"ATi9pQF_eqb99UK84R5rq8lGfRGpilVQOYyth7rXxh8">>, - Opts), - assert_item_read( - <<"4VSfUbhMVZQHW5VfVwQZOmC5fR3W21DZgFCyz8CA-cE">>, - Opts), - assert_item_read( - <<"ZQRHZhktk6dAtX9BlhO1teOtVlGHoyaWP25kAlhxrM4">>, - Opts), + hb_pmap:parallel_map( + [ + <<"0vy2Ey8bWkSDcRIvWQJjxDeVGYOrTSmYIIhBILJntY8">>, + <<"2lmrYydmDweX2MgGH39ZEB9hKm2JqGOYmRiG3n_xh8A">>, + <<"ATi9pQF_eqb99UK84R5rq8lGfRGpilVQOYyth7rXxh8">>, + <<"4VSfUbhMVZQHW5VfVwQZOmC5fR3W21DZgFCyz8CA-cE">>, + <<"ZQRHZhktk6dAtX9BlhO1teOtVlGHoyaWP25kAlhxrM4">> + ], + fun(ItemID) -> assert_item_read(ItemID, Opts) end, + 5 + ), % The T2pluNnaavL7-S2GkO_m3pASLUqMH_XQ9IiIhZKfySs can be deserialized so % we'll verify that some of its items were index and match the version % in the deserialized bundle. @@ -996,11 +992,12 @@ setup_index_opts() -> assert_bundle_read(BundleID, ExpectedItems, Opts) -> ReadItems = - lists:map( + hb_pmap:parallel_map( + ExpectedItems, fun({ItemID, _Index}) -> assert_item_read(ItemID, Opts) end, - ExpectedItems + length(ExpectedItems) ), Bundle = assert_item_read(BundleID, Opts), lists:foreach( diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index 5c9e996e4..e832517e9 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -61,7 +61,8 @@ do_request(Args, Opts) -> case hb_opts:get(http_client, ?DEFAULT_HTTP_CLIENT, Opts) of gun -> gun_req(Args, Opts); httpc -> httpc_req(Args, Opts); - hackney -> hackney_req(Args, Opts) + hackney -> hackney_req(Args, Opts); + Module -> Module:request(Args, Opts) end. maybe_retry(0, _, OriginalResponse, _) -> OriginalResponse; diff --git a/src/hb_test_cache.erl b/src/hb_test_cache.erl new file mode 100644 index 000000000..bc3018fce --- /dev/null +++ b/src/hb_test_cache.erl @@ -0,0 +1,86 @@ +%%% @doc Transparent HTTP response cache for tests. Activated by setting +%%% `http_client' to `hb_test_cache' (the test rebar3 profile does this +%%% automatically via DEFAULT_HTTP_CLIENT). Caches successful GET responses +%%% to external hosts on disk so repeated test runs avoid redundant network +%%% round-trips. +-module(hb_test_cache). +-export([request/2]). + +-define(CACHE_DIR, "/tmp/hb_test_cache/"). + +%% @doc Handle an HTTP request: serve from cache when possible, otherwise +%% delegate to hb_http_client and cache the result. +request(Args, Opts) -> + #{method := Method, peer := Peer, path := Path} = Args, + case is_cacheable(Method, Peer) of + false -> + hb_http_client:request(Args, Opts#{http_client => hackney}); + true -> + Key = cache_key(Path), + case read_cache(Key) of + {ok, Response} -> + io:put_chars(standard_error, + io_lib:format("[cache] HIT ~s~s~n", [Peer, Path])), + Response; + miss -> + Response = + hb_http_client:request( + Args, + Opts#{http_client => hackney} + ), + write_cache(Key, Response), + Response + end + end. + +%% @doc Return true when the request is a GET to a non-local peer. +is_cacheable(Method, Peer) -> + is_get(Method) andalso is_external(Peer). + +%% @doc Match common representations of the GET method. +is_get(<<"GET">>) -> true; +is_get(<<"get">>) -> true; +is_get(get) -> true; +is_get(_) -> false. + +%% @doc Return false for localhost/loopback peers, true otherwise. +is_external(<<"http://localhost", _/binary>>) -> false; +is_external(<<"http://127.0.0.1", _/binary>>) -> false; +is_external("http://localhost" ++ _) -> false; +is_external("http://127.0.0.1" ++ _) -> false; +is_external(_) -> true. + +%% @doc Produce a hex-encoded SHA-256 hash of the path for use as filename. +cache_key(Path) -> + Hash = crypto:hash(sha256, to_bin(Path)), + lists:flatten( + [io_lib:format("~2.16.0b", [B]) || <> <= Hash] + ). + +%% @doc Coerce a list or binary to binary. +to_bin(B) when is_binary(B) -> B; +to_bin(L) when is_list(L) -> list_to_binary(L). + +%% @doc Read a cached response from disk. Returns {ok, Term} or miss. +read_cache(Key) -> + case file:read_file(cache_path(Key)) of + {ok, Data} -> + try {ok, binary_to_term(Data, [safe])} + catch _:_ -> miss + end; + {error, _} -> miss + end. + +%% @doc Write a successful response (status < 400) to the cache directory. +%% Non-cacheable responses are silently ignored. +write_cache(Key, {ok, Status, _, _} = Response) + when Status < 400 -> + Path = cache_path(Key), + filelib:ensure_dir(Path), + file:write_file(Path, term_to_binary(Response)); +write_cache(_, _) -> + ok. + +%% @doc Build the full filesystem path for a cache key. +cache_path(Key) -> + ?CACHE_DIR ++ Key ++ ".bin". diff --git a/src/include/hb_opts.hrl b/src/include/hb_opts.hrl index b15061513..1e3ba484a 100644 --- a/src/include/hb_opts.hrl +++ b/src/include/hb_opts.hrl @@ -1,2 +1,4 @@ +-ifndef(DEFAULT_HTTP_CLIENT). -define(DEFAULT_HTTP_CLIENT, hackney). +-endif.