Initial commit for ar_p3 rework

.github/workflows/test.yml

@@ -26,7 +26,7 @@ jobs:
       max-parallel: 12
       matrix:
         core_test_mod: [
-            ## Long-running tests. Put these first to limit the overall runtime of the 
+            ## Long-running tests. Put these first to limit the overall runtime of the
             ## test suite
             ar_coordinated_mining_tests,
             ar_data_sync_tests,
@@ -62,6 +62,7 @@ jobs:
             # ar_p3,
             # ar_p3_config,
             # ar_p3_db,
+            ar_p3_ledger,
             ar_packing_server,
             ar_patricia_tree,
             ar_peers,
@@ -99,6 +100,7 @@ jobs:
             ar_tx_blacklist_tests,
             ar_tx_replay_pool_tests,
             ar_vdf_tests,
+            ar_p3_ledger_tests,
           ]
     steps:
       - uses: actions/checkout@v4

apps/arweave/include/ar_config.hrl

@@ -114,6 +114,11 @@
 %% The default rocksdb WAL sync interval, 1 minute.
 -define(DEFAULT_ROCKSDB_WAL_SYNC_INTERVAL_S, 60).
 
+%% The default P3 ledger checkpoint interval, 100 records.
+-define(DEFAULT_P3_LEDGER_CHECKPOINT_INTERVAL, 100).
+%% The default P3 ledger shutdown timeout, 1 minute.
+-define(DEFAULT_P3_LEDGER_SHUTDOWN_TIMEOUT, 60).
+
 %% @doc Startup options with default values.
 -record(config, {
 	init = false,
@@ -210,7 +215,9 @@
 	%% Undocumented/unsupported options
 	chunk_storage_file_size = ?CHUNK_GROUP_SIZE,
 	rocksdb_flush_interval_s = ?DEFAULT_ROCKSDB_FLUSH_INTERVAL_S,
-	rocksdb_wal_sync_interval_s = ?DEFAULT_ROCKSDB_WAL_SYNC_INTERVAL_S
+	rocksdb_wal_sync_interval_s = ?DEFAULT_ROCKSDB_WAL_SYNC_INTERVAL_S,
+	p3_ledger_checkpoint_interval = ?DEFAULT_P3_LEDGER_CHECKPOINT_INTERVAL,
+	p3_ledger_shutdown_timeout = ?DEFAULT_P3_LEDGER_SHUTDOWN_TIMEOUT
 }).
 
 -endif.

apps/arweave/include/ar_p3.hrl

@@ -35,7 +35,8 @@
 
 -record(p3_config, {
 	payments = #{},
-	services = #{}
+	services = #{},
+	ledger = #{}
 }).
 
 -record(p3_account, {

apps/arweave/include/ar_p3_ledger.hrl

@@ -0,0 +1,26 @@
+-ifndef(AR_P3_LEDGER_HRL).
+
+-define(AR_P3_LEDGER_HRL, true).
+
+-define(ar_p3_ledger_record_type_in, debit).
+-define(ar_p3_ledger_record_type_out, credit).
+
+-define(ar_p3_ledger_record_account_tokens, tokens).
+-define(ar_p3_ledger_record_account_credits, credits).
+-define(ar_p3_ledger_record_account_services, services).
+
+-record(ar_p3_ledger_record_v1, {
+	id :: ar_p3_ledger:record_id(),
+	type :: ar_p3_ledger:record_type(),
+	counterpart :: ar_p3_ledger:record_account(),
+	asset :: binary(),
+	amount :: non_neg_integer(),
+	meta :: #{binary() := binary()}
+}).
+
+-record(ar_p3_ledger_checkpoint_v1, {
+	id :: ar_p3_ledger:record_id(),
+	assets_map :: ar_p3_ledger:assets_map()
+}).
+
+-endif.

apps/arweave/src/ar.erl

@@ -336,7 +336,9 @@ show_help() ->
 					"Useful if you have multiple machines (or replicas) "
 					"and you want to monitor them separately on pool"},
 			{"rocksdb_flush_interval", "RocksDB flush interval in seconds"},
-			{"rocksdb_wal_sync_interval", "RocksDB WAL sync interval in seconds"}
+			{"rocksdb_wal_sync_interval", "RocksDB WAL sync interval in seconds"},
+			{"p3_ledger_checkpoint_interval", "P3 ledger checkpoint interval in records"},
+			{"p3_ledger_shutdown_timeout", "P3 ledger individual ledger process shutdown timeout in seconds"}
 		]
 	),
 	erlang:halt().
@@ -631,6 +633,10 @@ parse_cli_args(["rocksdb_flush_interval", Seconds | Rest], C) ->
 	parse_cli_args(Rest, C#config{ rocksdb_flush_interval_s = list_to_integer(Seconds) });
 parse_cli_args(["rocksdb_wal_sync_interval", Seconds | Rest], C) ->
 	parse_cli_args(Rest, C#config{ rocksdb_wal_sync_interval_s = list_to_integer(Seconds) });
+parse_cli_args(["p3_ledger_checkpoint_interval", Interval | Rest], C) ->
+	parse_cli_args(Rest, C#config{ p3_ledger_checkpoint_interval = list_to_integer(Interval) });
+parse_cli_args(["p3_ledger_shutdown_timeout", Seconds | Rest], C) ->
+	parse_cli_args(Rest, C#config{ p3_ledger_shutdown_timeout = list_to_integer(Seconds) });
 
 %% Undocumented/unsupported options
 parse_cli_args(["chunk_storage_file_size", Num | Rest], C) ->

apps/arweave/src/ar_config.erl

@@ -656,6 +656,13 @@ parse_options([{<<"rocksdb_wal_sync_interval">>, IntervalS} | Rest], Config)
 parse_options([{<<"rocksdb_wal_sync_interval">>, IntervalS} | _], _) ->
 	{error, {bad_type, rocksdb_wal_sync_interval, number}, IntervalS};
 
+parse_options([{<<"p3_ledger_checkpoint_interval">>, IntervalRecords} | Rest], Config)
+  		when is_integer(IntervalRecords) andalso IntervalRecords > 0 ->
+	parse_options(Rest, Config#config{ p3_ledger_checkpoint_interval = IntervalRecords });
+parse_options([{<<"p3_ledger_shutdown_timeout">>, ShutdownTimeoutS} | Rest], Config)
+  		when is_integer(ShutdownTimeoutS) andalso ShutdownTimeoutS > 0 ->
+	parse_options(Rest, Config#config{ p3_ledger_shutdown_timeout = ShutdownTimeoutS });
+
 parse_options([Opt | _], _) ->
 	{error, unknown, Opt};
 parse_options([], Config) ->
@@ -937,4 +944,4 @@ validate_storage_modules(Config) ->
 			io:format("~nThe node cannot mine multiple packing difficulties "
 					"for the same mining address.~n~n"),
 			false
-	end.
+	end.

apps/arweave/src/ar_p3.erl

@@ -3,7 +3,6 @@
 -behaviour(gen_server).
 
 -include_lib("arweave/include/ar.hrl").
--include_lib("arweave/include/ar_config.hrl").
 -include_lib("arweave/include/ar_p3.hrl").
 
 -export([start_link/0, allow_request/1, reverse_charge/1, get_balance/3, get_rates_json/0]).
@@ -98,7 +97,7 @@ handle_info({event, node_state, {new_tip, B, _PrevB}}, State) ->
 handle_info({event, node_state, _}, State) ->
 	{noreply, State}.
 
-terminate(Reason, State) ->
+terminate(_Reason, _State) ->
 	ok.
 
 %%%===================================================================
@@ -206,7 +205,7 @@ validate_signature(DecodedAddress, Req) ->
 	case ar_util:safe_decode(cowboy_req:header(?P3_SIGNATURE_HEADER, Req)) of
 		{ok, DecodedSignature} ->
 			validate_signature(DecodedAddress, DecodedSignature, Req);
-		Result ->
+		_Result ->
 			{error, invalid_signature}
 	end.
 
@@ -221,7 +220,7 @@ validate_signature(DecodedAddress, DecodedSignature, Req) ->
 				false ->
 					{error, invalid_signature}
 			end;
-		Error ->
+		_Error ->
 			{error, invalid_signature}
 	end.