WIP merkle tree rebase support

apps/arweave/include/ar.hrl

@@ -563,7 +563,19 @@
 	%% The proof of signing the same block several times or extending two equal forks.
 	double_signing_proof,
 	%% The cumulative difficulty of the previous block.
-	previous_cumulative_diff = 0
+	previous_cumulative_diff = 0,
+
+	%%
+	%% The fields below were added at the fork 2.7 (note that 2.6.8 was a hard fork too).
+	%%
+
+	%% The merkle trees of the data written after this weave offset may be constructed
+	%% in a way where some subtrees are "rebased", i.e., their offsets start from 0 as if
+	%% they were the leftmost subtree of the entire tree. The merkle paths for the chunks
+	%% belonging to the subtrees will include a 32-byte 0-sequence preceding the pivot to
+	%% the corresponding subtree. The rebases allow for flexible combination of data before
+	%% registering it on the weave, extremely useful e.g., for the bundling services.
+	merkle_rebase_support_threshold
 }).
 
 %% @doc A transaction.

apps/arweave/src/ar_block.erl

@@ -206,7 +206,8 @@ generate_signed_hash(#block{ previous_block = PrevH, timestamp = TS,
 		kryder_plus_rate_multiplier = KryderPlusRateMultiplier,
 		kryder_plus_rate_multiplier_latch = KryderPlusRateMultiplierLatch,
 		denomination = Denomination, redenomination_height = RedenominationHeight,
-		double_signing_proof = DoubleSigningProof, previous_cumulative_diff = PrevCDiff }) ->
+		double_signing_proof = DoubleSigningProof, previous_cumulative_diff = PrevCDiff,
+		merkle_rebase_support_threshold = RebaseThreshold }) ->
 	GetTXID = fun(TXID) when is_binary(TXID) -> TXID; (TX) -> TX#tx.id end,
 	Nonce2 = binary:encode_unsigned(Nonce),
 	%% The only block where reward_address may be unclaimed
@@ -218,6 +219,13 @@ generate_signed_hash(#block{ previous_block = PrevH, timestamp = TS,
 			next_partition_upper_bound = NextPartitionUpperBound,
 			steps = Steps, prev_output = PrevOutput,
 			last_step_checkpoints = LastStepCheckpoints } = NonceLimiterInfo,
+	RebaseThresholdBin =
+		case Height >= ar_fork:height_2_7() of
+			true ->
+				encode_int(RebaseThreshold, 16);
+			false ->
+				<<>>
+		end,
 	Segment = << (encode_bin(PrevH, 8))/binary, (encode_int(TS, 8))/binary,
 			(encode_bin(Nonce2, 16))/binary, (encode_int(Height, 8))/binary,
 			(encode_int(Diff, 16))/binary, (encode_int(CDiff, 16))/binary,
@@ -227,8 +235,10 @@ generate_signed_hash(#block{ previous_block = PrevH, timestamp = TS,
 			(encode_bin(WalletList, 8))/binary,
 			(encode_bin(HashListMerkle, 8))/binary, (encode_int(RewardPool, 8))/binary,
 			(encode_int(Packing_2_5_Threshold, 8))/binary,
-			(encode_int(StrictChunkThreshold, 8))/binary, (encode_int(RateDividend, 8))/binary,
-			(encode_int(RateDivisor, 8))/binary, (encode_int(ScheduledRateDividend, 8))/binary,
+			(encode_int(StrictChunkThreshold, 8))/binary,
+					(encode_int(RateDividend, 8))/binary,
+			(encode_int(RateDivisor, 8))/binary,
+					(encode_int(ScheduledRateDividend, 8))/binary,
 			(encode_int(ScheduledRateDivisor, 8))/binary,
 			(encode_bin_list(Tags, 16, 16))/binary,
 			(encode_bin_list([GetTXID(TX) || TX <- TXs], 16, 8))/binary,
@@ -247,7 +257,7 @@ generate_signed_hash(#block{ previous_block = PrevH, timestamp = TS,
 			KryderPlusRateMultiplier:24, KryderPlusRateMultiplierLatch:8, Denomination:24,
 			(encode_int(RedenominationHeight, 8))/binary,
 			(ar_serialize:encode_double_signing_proof(DoubleSigningProof))/binary,
-			(encode_int(PrevCDiff, 16))/binary >>,
+			(encode_int(PrevCDiff, 16))/binary, RebaseThresholdBin/binary >>,
 	crypto:hash(sha256, Segment).
 
 %% @doc Compute the block identifier from the signed hash and block signature.

apps/arweave/src/ar_block_cache.erl

@@ -559,9 +559,9 @@ prune(Tab, Depth, TipHeight) ->
 	end.
 
 update_longest_chain_cache(Tab) ->
-	GetStart = erlang:timestamp(),
 	[{_, {_CDiff, H}}] = ets:lookup(Tab, max_cdiff),
-	Result = get_longest_chain_block_txs_pairs(Tab, H, ?STORE_BLOCKS_BEHIND_CURRENT, none, none, [], 0),
+	Result = get_longest_chain_block_txs_pairs(Tab, H, ?STORE_BLOCKS_BEHIND_CURRENT,
+			none, none, [], 0),
 	case ets:update_element(Tab, longest_chain, {2, Result}) of
 		true -> ok;
 		false ->
@@ -848,11 +848,6 @@ random_block_after_repacking(CDiff) ->
 	#block{ indep_hash = crypto:strong_rand_bytes(48), height = 0, cumulative_diff = CDiff,
 			hash = crypto:strong_rand_bytes(32) }.
 
-random_block_with_txs(CDiff, NumTXs) ->
-	TXs = [ #tx{ id = crypto:strong_rand_bytes(32) } || _ <- lists:seq(1, NumTXs)],
-	B = random_block(CDiff),
-	B#block{ txs = TXs }.
-
 block_id(#block{ indep_hash = H }) ->
 	H.