Publish indexed transactions with BLAKE2b hashes to IPFS DHT

cumulus/client/pov-recovery/Cargo.toml

@@ -24,6 +24,7 @@ sc-consensus = { workspace = true, default-features = true }
 sc-network = { workspace = true, default-features = true }
 sp-api = { workspace = true, default-features = true }
 sp-consensus = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
 sp-maybe-compressed-blob = { workspace = true, default-features = true }
 sp-runtime = { workspace = true, default-features = true }
 sp-version = { workspace = true, default-features = true }

cumulus/client/pov-recovery/src/tests.rs

@@ -42,6 +42,7 @@ use sc_consensus::import_queue::RuntimeOrigin;
 use sc_utils::mpsc::{TracingUnboundedReceiver, TracingUnboundedSender};
 use sp_api::RuntimeApiInfo;
 use sp_blockchain::Info;
+use sp_core::H256;
 use sp_runtime::{generic::SignedBlock, Justifications};
 use sp_version::RuntimeVersion;
 use std::{
@@ -198,11 +199,15 @@ impl<Block: BlockT> BlockBackend<Block> for ParachainClient<Block> {
 		unimplemented!()
 	}
 
-	fn indexed_transaction(&self, _: Block::Hash) -> sp_blockchain::Result<Option<Vec<u8>>> {
+	fn indexed_transaction(&self, _: H256) -> sp_blockchain::Result<Option<Vec<u8>>> {
 		unimplemented!()
 	}
 
-	fn has_indexed_transaction(&self, _: Block::Hash) -> sp_blockchain::Result<bool> {
+	fn has_indexed_transaction(&self, _: H256) -> sp_blockchain::Result<bool> {
+		unimplemented!()
+	}
+
+	fn block_indexed_hashes(&self, _: Block::Hash) -> sp_blockchain::Result<Option<Vec<H256>>> {
 		unimplemented!()
 	}
 

cumulus/client/relay-chain-minimal-node/src/network.rs

@@ -76,7 +76,7 @@ pub(crate) fn build_collator_network<Network: NetworkBackend<Block, Hash>>(
 		protocol_id,
 		metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()),
 		block_announce_config,
-		bitswap_config: None,
+		ipfs_config: None,
 		notification_metrics,
 	};
 

prdoc/pr_10468.prdoc

@@ -0,0 +1,28 @@
+title: Publish indexed transactions with BLAKE2b hashes to IPFS DHT
+doc:
+- audience: [Node Dev, Node Operator]
+  description: Add `--ipfs-bootnodes` flag for specifying IPFS bootnodes. If passed
+    along with `--ipfs-server`, the node will register as a content provider in IPFS
+    DHT of indexed transactions with BLAKE2b hashes of the last two weeks (or pruning
+    depth if smaller).
+crates:
+- name: sc-network
+  bump: major
+- name: sc-network-types
+  bump: minor
+- name: cumulus-client-pov-recovery
+  bump: minor
+- name: sc-client-api
+  bump: major
+- name: sc-client-db
+  bump: minor
+- name: sc-rpc-spec-v2
+  bump: patch
+- name: sc-service
+  bump: major
+- name: sp-blockchain
+  bump: minor
+- name: cumulus-relay-chain-minimal-node
+  bump: patch
+- name: sc-cli
+  bump: major

substrate/client/api/src/client.rs

@@ -19,7 +19,7 @@
 //! A set of APIs supported by the client along with their primitives.
 
 use sp_consensus::BlockOrigin;
-use sp_core::storage::StorageKey;
+use sp_core::{storage::StorageKey, H256};
 use sp_runtime::{
 	generic::SignedBlock,
 	traits::{Block as BlockT, NumberFor},
@@ -132,13 +132,19 @@ pub trait BlockBackend<Block: BlockT> {
 		hash: Block::Hash,
 	) -> sp_blockchain::Result<Option<Vec<<Block as BlockT>::Extrinsic>>>;
 
-	/// Get all indexed transactions for a block,
-	/// including renewed transactions.
+	/// Get all indexed transactions for a block, including renewed transactions.
 	///
-	/// Note that this will only fetch transactions
-	/// that are indexed by the runtime with `storage_index_transaction`.
+	/// Note that this will only fetch transactions that are indexed by the runtime with
+	/// `storage_index_transaction`.
 	fn block_indexed_body(&self, hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>>;
 
+	/// Get the BLAKE2b-256 hashes of all indexed transactions in a block, including renewed
+	/// transactions.
+	///
+	/// Note that this will only fetch transactions that are indexed by the runtime with
+	/// `storage_index_transaction`.
+	fn block_indexed_hashes(&self, hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<H256>>>;
+
 	/// Get full block by hash.
 	fn block(&self, hash: Block::Hash) -> sp_blockchain::Result<Option<SignedBlock<Block>>>;
 
@@ -151,14 +157,14 @@ pub trait BlockBackend<Block: BlockT> {
 	/// Get block hash by number.
 	fn block_hash(&self, number: NumberFor<Block>) -> sp_blockchain::Result<Option<Block::Hash>>;
 
-	/// Get single indexed transaction by content hash.
+	/// Get single indexed transaction by content hash (BLAKE2b-256).
 	///
 	/// Note that this will only fetch transactions
 	/// that are indexed by the runtime with `storage_index_transaction`.
-	fn indexed_transaction(&self, hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<u8>>>;
+	fn indexed_transaction(&self, hash: H256) -> sp_blockchain::Result<Option<Vec<u8>>>;
 
-	/// Check if transaction index exists.
-	fn has_indexed_transaction(&self, hash: Block::Hash) -> sp_blockchain::Result<bool> {
+	/// Check if transaction index exists given its BLAKE2b-256 hash.
+	fn has_indexed_transaction(&self, hash: H256) -> sp_blockchain::Result<bool> {
 		Ok(self.indexed_transaction(hash)?.is_some())
 	}
 

substrate/client/api/src/in_mem.rs

@@ -21,7 +21,7 @@
 use parking_lot::RwLock;
 use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata};
 use sp_core::{
-	offchain::storage::InMemOffchainStorage as OffchainStorage, storage::well_known_keys,
+	offchain::storage::InMemOffchainStorage as OffchainStorage, storage::well_known_keys, H256,
 };
 use sp_runtime::{
 	generic::BlockId,
@@ -423,7 +423,11 @@ impl<Block: BlockT> blockchain::Backend<Block> for Blockchain<Block> {
 		unimplemented!()
 	}
 
-	fn indexed_transaction(&self, _hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<u8>>> {
+	fn indexed_transaction(&self, _hash: H256) -> sp_blockchain::Result<Option<Vec<u8>>> {
+		unimplemented!("Not supported by the in-mem backend.")
+	}
+
+	fn block_indexed_hashes(&self, _hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<H256>>> {
 		unimplemented!("Not supported by the in-mem backend.")
 	}
 

substrate/client/cli/src/params/network_params.rs

@@ -153,6 +153,10 @@ pub struct NetworkParams {
 	#[arg(long)]
 	pub ipfs_server: bool,
 
+	/// Specify a list of IPFS bootstrap nodes.
+	#[arg(long, value_name = "ADDR", num_args = 1.., requires = "ipfs_server")]
+	pub ipfs_bootnodes: Vec<MultiaddrWithPeerId>,
+
 	/// Blockchain syncing mode.
 	#[arg(
 		long,
@@ -283,6 +287,7 @@ impl NetworkParams {
 			kademlia_disjoint_query_paths: self.kademlia_disjoint_query_paths,
 			kademlia_replication_factor: self.kademlia_replication_factor,
 			ipfs_server: self.ipfs_server,
+			ipfs_bootnodes: self.ipfs_bootnodes.clone(),
 			sync_mode: self.sync.into(),
 			network_backend: self.network_backend.into(),
 		}

substrate/client/db/src/lib.rs

@@ -665,6 +665,29 @@ impl<Block: BlockT> BlockchainDb<Block> {
 		}
 		Ok(None)
 	}
+
+	fn block_indexed_hashes_iter(
+		&self,
+		hash: Block::Hash,
+	) -> ClientResult<Option<impl Iterator<Item = DbHash>>> {
+		let body = match read_db(
+			&*self.db,
+			columns::KEY_LOOKUP,
+			columns::BODY_INDEX,
+			BlockId::<Block>::Hash(hash),
+		)? {
+			Some(body) => body,
+			None => return Ok(None),
+		};
+		match Vec::<DbExtrinsic<Block>>::decode(&mut &body[..]) {
+			Ok(index) => Ok(Some(index.into_iter().flat_map(|ex| match ex {
+				DbExtrinsic::Indexed { hash, .. } => Some(hash),
+				_ => None,
+			}))),
+			Err(err) =>
+				Err(sp_blockchain::Error::Backend(format!("Error decoding body list: {err}"))),
+		}
+	}
 }
 
 impl<Block: BlockT> sc_client_api::blockchain::HeaderBackend<Block> for BlockchainDb<Block> {
@@ -750,42 +773,32 @@ impl<Block: BlockT> sc_client_api::blockchain::Backend<Block> for BlockchainDb<B
 		children::read_children(&*self.db, columns::META, meta_keys::CHILDREN_PREFIX, parent_hash)
 	}
 
-	fn indexed_transaction(&self, hash: Block::Hash) -> ClientResult<Option<Vec<u8>>> {
+	fn indexed_transaction(&self, hash: DbHash) -> ClientResult<Option<Vec<u8>>> {
 		Ok(self.db.get(columns::TRANSACTION, hash.as_ref()))
 	}
 
-	fn has_indexed_transaction(&self, hash: Block::Hash) -> ClientResult<bool> {
+	fn has_indexed_transaction(&self, hash: DbHash) -> ClientResult<bool> {
 		Ok(self.db.contains(columns::TRANSACTION, hash.as_ref()))
 	}
 
+	fn block_indexed_hashes(&self, hash: Block::Hash) -> ClientResult<Option<Vec<DbHash>>> {
+		self.block_indexed_hashes_iter(hash).map(|hashes| hashes.map(Iterator::collect))
+	}
+
 	fn block_indexed_body(&self, hash: Block::Hash) -> ClientResult<Option<Vec<Vec<u8>>>> {
-		let body = match read_db(
-			&*self.db,
-			columns::KEY_LOOKUP,
-			columns::BODY_INDEX,
-			BlockId::<Block>::Hash(hash),
-		)? {
-			Some(body) => body,
-			None => return Ok(None),
-		};
-		match Vec::<DbExtrinsic<Block>>::decode(&mut &body[..]) {
-			Ok(index) => {
-				let mut transactions = Vec::new();
-				for ex in index.into_iter() {
-					if let DbExtrinsic::Indexed { hash, .. } = ex {
-						match self.db.get(columns::TRANSACTION, hash.as_ref()) {
-							Some(t) => transactions.push(t),
-							None =>
-								return Err(sp_blockchain::Error::Backend(format!(
-									"Missing indexed transaction {hash:?}",
-								))),
-						}
-					}
-				}
-				Ok(Some(transactions))
-			},
-			Err(err) =>
-				Err(sp_blockchain::Error::Backend(format!("Error decoding body list: {err}"))),
+		match self.block_indexed_hashes_iter(hash) {
+			Ok(Some(hashes)) => Ok(Some(
+				hashes
+					.map(|hash| match self.db.get(columns::TRANSACTION, hash.as_ref()) {
+						Some(t) => Ok(t),
+						None => Err(sp_blockchain::Error::Backend(format!(
+							"Missing indexed transaction {hash:?}",
+						))),
+					})
+					.collect::<Result<_, _>>()?,
+			)),
+			Ok(None) => Ok(None),
+			Err(err) => Err(err),
 		}
 	}
 }

substrate/client/network/Cargo.toml

@@ -30,6 +30,7 @@ async-channel = { workspace = true }
 async-trait = { workspace = true }
 asynchronous-codec = { workspace = true }
 bytes = { workspace = true, default-features = true }
+cid = { workspace = true }
 codec = { features = ["derive"], workspace = true, default-features = true }
 either = { workspace = true, default-features = true }
 fnv = { workspace = true }
@@ -69,7 +70,6 @@ zeroize = { workspace = true, default-features = true }
 
 [dev-dependencies]
 assert_matches = { workspace = true }
-cid = { workspace = true }
 multistream-select = { workspace = true }
 sc-block-builder = { workspace = true, default-features = true }
 sp-consensus = { workspace = true, default-features = true }

substrate/client/network/benches/notifications_protocol.rs

@@ -90,7 +90,7 @@ where
 		protocol_id: ProtocolId::from("bench-protocol-name"),
 		fork_id: None,
 		metrics_registry: None,
-		bitswap_config: None,
+		ipfs_config: None,
 		notification_metrics: NotificationMetrics::new(None),
 	})
 	.unwrap();

substrate/client/network/benches/request_response_protocol.rs

@@ -104,7 +104,7 @@ where
 		protocol_id: ProtocolId::from("bench-request-response-protocol"),
 		fork_id: None,
 		metrics_registry: None,
-		bitswap_config: None,
+		ipfs_config: None,
 		notification_metrics: NotificationMetrics::new(None),
 	})
 	.unwrap();

substrate/client/network/src/bitswap/mod.rs

@@ -37,6 +37,7 @@ use schema::bitswap::{
 	message::{wantlist::WantType, Block as MessageBlock, BlockPresence, BlockPresenceType},
 	Message as BitswapMessage,
 };
+use sp_core::H256;
 use sp_runtime::traits::Block as BlockT;
 use std::{io, sync::Arc, time::Duration};
 use unsigned_varint::encode as varint_encode;
@@ -207,7 +208,7 @@ impl<B: BlockT> BitswapRequestHandler<B> {
 				continue
 			}
 
-			let mut hash = B::Hash::default();
+			let mut hash = H256::default();
 			hash.as_mut().copy_from_slice(&cid.hash().digest()[0..32]);
 			let transaction = match self.client.indexed_transaction(hash) {
 				Ok(ex) => ex,

substrate/client/network/src/config.rs

@@ -676,9 +676,12 @@ pub struct NetworkConfiguration {
 	/// `kademlia_replication_factor` peers to consider record successfully put.
 	pub kademlia_replication_factor: NonZeroUsize,
 
-	/// Enable serving block data over IPFS bitswap.
+	/// Enable publishing of indexed transactions to IPFS.
 	pub ipfs_server: bool,
 
+	/// List of IPFS bootstrap nodes.
+	pub ipfs_bootnodes: Vec<MultiaddrWithPeerId>,
+
 	/// Networking backend used for P2P communication.
 	pub network_backend: NetworkBackendType,
 }
@@ -714,6 +717,7 @@ impl NetworkConfiguration {
 			kademlia_replication_factor: NonZeroUsize::new(DEFAULT_KADEMLIA_REPLICATION_FACTOR)
 				.expect("value is a constant; constant is non-zero; qed."),
 			ipfs_server: false,
+			ipfs_bootnodes: Vec::new(),
 			network_backend: NetworkBackendType::Litep2p,
 		}
 	}
@@ -749,6 +753,16 @@ impl NetworkConfiguration {
 	}
 }
 
+/// IPFS server configuration.
+pub struct IpfsConfig<Block: BlockT, H: ExHashT, N: NetworkBackend<Block, H>> {
+	/// Network-backend-specific Bitswap configuration.
+	pub bitswap_config: N::BitswapConfig,
+	/// Indexed transactions provider.
+	pub block_provider: Box<dyn crate::IpfsBlockProvider>,
+	/// IPFS bootstrap nodes.
+	pub bootnodes: Vec<MultiaddrWithPeerId>,
+}
+
 /// Network initialization parameters.
 pub struct Params<Block: BlockT, H: ExHashT, N: NetworkBackend<Block, H>> {
 	/// Assigned role for our node (full, light, ...).
@@ -777,7 +791,7 @@ pub struct Params<Block: BlockT, H: ExHashT, N: NetworkBackend<Block, H>> {
 	pub block_announce_config: N::NotificationProtocolConfig,
 
 	/// Bitswap configuration, if the server has been enabled.
-	pub bitswap_config: Option<N::BitswapConfig>,
+	pub ipfs_config: Option<IpfsConfig<Block, H, N>>,
 
 	/// Notification metrics.
 	pub notification_metrics: NotificationMetrics,