Tests: run all modules in parallel and cache external http calls (unless explicitly set to an http client or opt out)

Makefile

@@ -1,8 +1,11 @@
-.PHONY: compile
+.PHONY: compile test
 
 compile:
 	rebar3 compile
 
+test:
+	scripts/parallel_tests.sh
+
 WAMR_VERSION = 2.2.0
 WAMR_DIR = _build/wamr
 

rebar.config

@@ -56,6 +56,7 @@
             {d, 'ENABLE_ROCKSDB', true}
         ]}
     ]},
+    {test, [{erl_opts, [{d, 'DEFAULT_HTTP_CLIENT', hb_test_cache}]}]},
     {http3, [
         {deps, [
             {quicer, {git, "https://github.com/emqx/quic.git",

scripts/parallel_tests.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+#
+# Run EUnit test modules in parallel across separate BEAM VMs.
+# Each VM gets a unique HB_PORT to avoid port conflicts.
+#
+# Usage:
+#   scripts/parallel_tests.sh                    # run all test modules
+#   scripts/parallel_tests.sh mod1 mod2 mod3     # run specific modules
+#
+set -uo pipefail
+
+NPROC=$(nproc 2>/dev/null || echo 16)
+BASE_PORT=19000
+EBIN_DIR="_build/test/lib/hb/ebin"
+PA_DIRS="_build/test/lib/*/ebin"
+LOG_DIR="/tmp/eunit_parallel"
+CFLAGS="${CFLAGS:--fpermissive}"
+export CFLAGS
+
+mkdir -p "$LOG_DIR"
+
+echo "=== Compiling (test profile) ==="
+rebar3 as test compile || { echo "Compilation failed"; exit 1; }
+
+if [ $# -gt 0 ]; then
+    MODULES="$*"
+else
+    echo "=== Discovering test modules ==="
+    MODULES=$(erl -noshell -pa $PA_DIRS -eval '
+        Beams = filelib:wildcard("'"$EBIN_DIR"'/*.beam"),
+        Mods = [list_to_atom(filename:basename(B, ".beam")) || B <- Beams],
+        TestMods = lists:filter(fun(M) ->
+            try
+                Exports = M:module_info(exports),
+                lists:any(fun({F,0}) ->
+                    S = atom_to_list(F),
+                    lists:suffix("_test", S) orelse lists:suffix("_test_", S);
+                (_) -> false end, Exports)
+            catch _:_ -> false
+            end
+        end, Mods),
+        [io:format("~s~n", [M]) || M <- lists:sort(TestMods)],
+        halt(0).
+    ' 2>/dev/null)
+fi
+
+MODULE_COUNT=$(echo "$MODULES" | wc -w)
+echo "=== Running $MODULE_COUNT modules with $NPROC workers ==="
+echo ""
+
+T_START=$(date +%s)
+
+run_module() {
+    local mod=$1
+    local port=$2
+    local logfile="$LOG_DIR/${mod}.log"
+
+    local t0
+    t0=$(date +%s%3N 2>/dev/null || python3 -c 'import time; print(int(time.time()*1000))')
+
+    HB_PORT=$port erl -noshell -pa $PA_DIRS -eval "
+        application:ensure_all_started(hb),
+        case eunit:test($mod, [verbose]) of
+            ok -> halt(0);
+            error -> halt(1)
+        end.
+    " > "$logfile" 2>&1
+    local rc=$?
+
+    local t1
+    t1=$(date +%s%3N 2>/dev/null || python3 -c 'import time; print(int(time.time()*1000))')
+    local elapsed=$(( (t1 - t0) / 1000 ))
+
+    if [ $rc -eq 0 ]; then
+        printf "  \033[32mPASS\033[0m  %-45s %3ds\n" "$mod" "$elapsed"
+    else
+        printf "  \033[31mFAIL\033[0m  %-45s %3ds  (see %s)\n" "$mod" "$elapsed" "$logfile"
+    fi
+    return $rc
+}
+
+export -f run_module
+export PA_DIRS LOG_DIR
+
+FAILED=0
+PORT=$BASE_PORT
+PIDS=()
+MODS_ARRAY=()
+
+for mod in $MODULES; do
+    run_module "$mod" "$PORT" &
+    PIDS+=($!)
+    MODS_ARRAY+=("$mod")
+    PORT=$((PORT + 1))
+
+    # Throttle to NPROC concurrent jobs
+    if [ ${#PIDS[@]} -ge "$NPROC" ]; then
+        wait "${PIDS[0]}" || FAILED=$((FAILED + 1))
+        PIDS=("${PIDS[@]:1}")
+        MODS_ARRAY=("${MODS_ARRAY[@]:1}")
+    fi
+done
+
+for pid in "${PIDS[@]}"; do
+    wait "$pid" || FAILED=$((FAILED + 1))
+done
+
+T_END=$(date +%s)
+ELAPSED=$((T_END - T_START))
+
+echo ""
+echo "=== Done in ${ELAPSED}s ==="
+echo "    Modules: $MODULE_COUNT"
+echo "    Failed:  $FAILED"
+
+if [ "$FAILED" -gt 0 ]; then
+    echo ""
+    echo "Failed module logs in $LOG_DIR/"
+    exit 1
+fi

src/dev_arweave_offset.erl

@@ -353,39 +353,27 @@ parse_offset_test() ->
 
 offset_item_cases_test() ->
     Opts = #{},
-    % A simple message.
-    assert_offset_item(
-        <<"160399272861859">>,
-        498852,
-        #{ <<"content-type">> => <<"image/png">> },
-        Opts
-    ),
-    % A reference with a given length.
-    assert_offset_item(
-        <<"160399272861859-498852">>,
-        498852,
-        #{ <<"content-type">> => <<"image/png">> },
-        Opts
-    ),
-    % A reference to a byte in the middle of the test message.
-    assert_offset_item(
-        <<"160399273000000">>,
-        498852,
-        #{ <<"content-type">> => <<"image/png">> },
-        Opts
-    ),
-    % A megabyte reference to the item, occurring in the middle of the item.
-    assert_offset_item(
-        <<"160399273m">>,
-        498852,
-        #{ <<"content-type">> => <<"image/png">> },
-        Opts
-    ),
-    assert_offset_item(
-        <<"384600234780716">>,
-        856691,
-        #{ <<"content-type">> => <<"image/jpeg">> },
-        Opts
+    hb_pmap:parallel_map(
+        [
+            % A simple message.
+            {<<"160399272861859">>, 498852,
+                #{<<"content-type">> => <<"image/png">>}},
+            % A reference with a given length.
+            {<<"160399272861859-498852">>, 498852,
+                #{<<"content-type">> => <<"image/png">>}},
+            % A reference to a byte in the middle of the test message.
+            {<<"160399273000000">>, 498852,
+                #{<<"content-type">> => <<"image/png">>}},
+            % A megabyte reference to the item, occurring in the middle of the item.
+            {<<"160399273m">>, 498852,
+                #{<<"content-type">> => <<"image/png">>}},
+            {<<"384600234780716">>, 856691,
+                #{<<"content-type">> => <<"image/jpeg">>}}
+        ],
+        fun({Path, DataSize, Tags}) ->
+            assert_offset_item(Path, DataSize, Tags, Opts)
+        end,
+        5
     ),
     ok.
 

src/dev_copycat_arweave.erl

@@ -480,21 +480,17 @@ index_ids_test() ->
             <<"WbRAQbeyjPHgopBKyi0PLeKWvYZr3rgZvQ7QY3ASJS4">>
         )
     ),
-    assert_item_read(
-        <<"0vy2Ey8bWkSDcRIvWQJjxDeVGYOrTSmYIIhBILJntY8">>,
-        Opts),
-    assert_item_read(
-        <<"2lmrYydmDweX2MgGH39ZEB9hKm2JqGOYmRiG3n_xh8A">>,
-        Opts),
-    assert_item_read(
-        <<"ATi9pQF_eqb99UK84R5rq8lGfRGpilVQOYyth7rXxh8">>,
-        Opts),
-    assert_item_read(
-        <<"4VSfUbhMVZQHW5VfVwQZOmC5fR3W21DZgFCyz8CA-cE">>,
-        Opts),
-    assert_item_read(
-        <<"ZQRHZhktk6dAtX9BlhO1teOtVlGHoyaWP25kAlhxrM4">>,
-        Opts),
+    hb_pmap:parallel_map(
+        [
+            <<"0vy2Ey8bWkSDcRIvWQJjxDeVGYOrTSmYIIhBILJntY8">>,
+            <<"2lmrYydmDweX2MgGH39ZEB9hKm2JqGOYmRiG3n_xh8A">>,
+            <<"ATi9pQF_eqb99UK84R5rq8lGfRGpilVQOYyth7rXxh8">>,
+            <<"4VSfUbhMVZQHW5VfVwQZOmC5fR3W21DZgFCyz8CA-cE">>,
+            <<"ZQRHZhktk6dAtX9BlhO1teOtVlGHoyaWP25kAlhxrM4">>
+        ],
+        fun(ItemID) -> assert_item_read(ItemID, Opts) end,
+        5
+    ),
     % The T2pluNnaavL7-S2GkO_m3pASLUqMH_XQ9IiIhZKfySs can be deserialized so
     % we'll verify that some of its items were index and match the version
     % in the deserialized bundle.
@@ -996,11 +992,12 @@ setup_index_opts() ->
 
 assert_bundle_read(BundleID, ExpectedItems, Opts) ->
     ReadItems =
-        lists:map(
+        hb_pmap:parallel_map(
+            ExpectedItems,
             fun({ItemID, _Index}) ->
                 assert_item_read(ItemID, Opts)
             end,
-            ExpectedItems
+            length(ExpectedItems)
         ),
     Bundle = assert_item_read(BundleID, Opts),
     lists:foreach(

src/hb_http_client.erl

@@ -63,7 +63,8 @@ request(Args, RemainingRetries, Opts) ->
 do_request(Args, Opts) ->
     case hb_opts:get(http_client, ?DEFAULT_HTTP_CLIENT, Opts) of
         gun -> gun_req(Args, Opts);
-        httpc -> httpc_req(Args, Opts)
+        httpc -> httpc_req(Args, Opts);
+        Module -> Module:request(Args, Opts)
     end.
 
 maybe_retry(0, _, OriginalResponse, _) -> OriginalResponse;

src/hb_test_cache.erl

@@ -0,0 +1,63 @@
+-module(hb_test_cache).
+-export([request/2]).
+
+-define(CACHE_DIR, "/tmp/hb_test_cache/").
+
+request(Args, Opts) ->
+    #{method := Method, peer := Peer, path := Path} = Args,
+    case is_cacheable(Method, Peer) of
+        false ->
+            hb_http_client:request(Args, Opts#{http_client => gun});
+        true ->
+            Key = cache_key(Path),
+            case read_cache(Key) of
+                {ok, Response} ->
+                    io:put_chars(standard_error,
+                        io_lib:format("[cache] HIT ~s~s~n", [Peer, Path])),
+                    Response;
+                miss ->
+                    Response = hb_http_client:request(Args, Opts#{http_client => gun}),
+                    maybe_write_cache(Key, Response),
+                    Response
+            end
+    end.
+
+is_cacheable(Method, Peer) ->
+    is_get(Method) andalso is_external(Peer).
+
+is_get(<<"GET">>) -> true;
+is_get(<<"get">>) -> true;
+is_get(get) -> true;
+is_get(_) -> false.
+
+is_external(<<"http://localhost", _/binary>>) -> false;
+is_external(<<"http://127.0.0.1", _/binary>>) -> false;
+is_external("http://localhost" ++ _) -> false;
+is_external("http://127.0.0.1" ++ _) -> false;
+is_external(_) -> true.
+
+cache_key(Path) ->
+    Hash = crypto:hash(sha256, to_bin(Path)),
+    lists:flatten([io_lib:format("~2.16.0b", [B]) || <<B>> <= Hash]).
+
+to_bin(B) when is_binary(B) -> B;
+to_bin(L) when is_list(L) -> list_to_binary(L).
+
+read_cache(Key) ->
+    case file:read_file(cache_path(Key)) of
+        {ok, Data} ->
+            try {ok, binary_to_term(Data)}
+            catch _:_ -> miss
+            end;
+        {error, _} -> miss
+    end.
+
+maybe_write_cache(Key, {ok, Status, _, _} = Response) when Status < 400 ->
+    Path = cache_path(Key),
+    filelib:ensure_dir(Path),
+    file:write_file(Path, term_to_binary(Response));
+maybe_write_cache(_, _) ->
+    ok.
+
+cache_path(Key) ->
+    ?CACHE_DIR ++ Key ++ ".bin".

src/include/hb_opts.hrl

@@ -1,2 +1,4 @@
+-ifndef(DEFAULT_HTTP_CLIENT).
 -define(DEFAULT_HTTP_CLIENT, gun).
+-endif.