* Introduce RPC and new protocol crate

* Fork ledger traits
* Genesis config for opa
* Opa administration logic / pallet
* Subxt based client - connectivity
* Development runtime - working, global logger use needs to consider rust::tracing
* Signature and address types, substrate crypto used directly for account signatures
* Seed mechanism changed in ChronicleSigning to allow us to supply a specific seed
* Remove transaction costs and balance checks

Signed-off-by: Ryan <[email protected]>

Cargo.toml

@@ -8,19 +8,21 @@ members = [
   "crates/chronicle-domain",
   "crates/chronicle-domain-lint",
   "crates/chronicle-domain-test",
-  "crates/chronicle-protocol",
   "crates/chronicle-synth",
   "crates/chronicle-signing",
   "crates/chronicle-telemetry",
   "crates/gq-subscribe",
   "crates/id-provider",
-  "crates/opa-tp",
-  "crates/opa-tp-protocol",
-  "crates/opactl",
-  "crates/sawtooth-tp",
   "crates/pallet-chronicle",
+  "crates/pallet-opa",
   "node/runtime-chronicle",
   "node/node-chronicle",
+  "crates/embedded-substrate",
+  "crates/runtime-api-chronicle",
+  "crates/protocol-abstract",
+  "crates/protocol-substrate",
+  "crates/protocol-substrate-chronicle",
+  "crates/protocol-substrate-opa",
 ]
 
 [workspace.dependencies]
@@ -29,16 +31,12 @@ anyhow = "1.0.6"
 assert_fs = "1.0"
 async-graphql = "5.0.9"
 async-graphql-poem = "5.0.9"
-async-stl-client = { git = "https://github.com/btpworks/async-stl-sdk" }
 async-stream = "0.3.3"
 async-trait = "0.1.61"
 backoff = { version = "0.4.0", features = ["futures", "tokio"] }
 base64 = "0.21"
 bytes = "1.3.0"
 cached = "0.42"
-frame-support = "24.0.0"
-frame-system = "24.0.0"
-frame-benchmarking  = "24.0.0"
 cfg-if = "1.0.0"
 chronicle-signing = { path = "crates/chronicle-signing" }
 chrono = "0.4.26"
@@ -53,14 +51,13 @@ criterion = { version = "0.5.1", features = ["async_futures", "async_tokio"] }
 crossbeam = "0.8.1"
 custom_error = "1.9.2"
 derivative = "2.2.0"
-diesel = { version = "2.0.0-rc.0", features = [
-  "postgres",
-  "uuid",
-  "chrono",
-  "r2d2",
-] }
-diesel_migrations = { version = "2.0.0-rc.0", features = ["postgres"] }
+diesel = { version = "2.1", features = ["postgres", "uuid", "chrono", "r2d2"] }
+diesel-async = { version = "0.4" }
+diesel_migrations = { version = "2.1", features = ["postgres"] }
 dotenvy = "0.15"
+frame-benchmarking = "24.0.0"
+frame-support = "24.0.0"
+frame-system = "24.0.0"
 futures = "0.3.21"
 genco = "0.16.1"
 glob = "0.3.0"
@@ -80,22 +77,24 @@ k256 = { version = "0.11.3", features = [
   "ecdsa",
   "pkcs8",
   "sha256",
+  "keccak256",
   "std",
   "pem",
   "serde",
 ] }
 lazy_static = "1.4.0"
 locspan = "0.7"
 lru = "0.11"
+macro-attr-2018 = "3.0.0"
 maplit = "1.0.2"
 metrics = "0.21.0"
 metrics-exporter-prometheus = "0.12.1"
 mime = "0.3"
+mobc = "0.8"
 mockito = "1.1"
+newtype-derive-2018 = "0.2.1"
 oauth2 = "4.4"
 opa = { git = "https://github.com/tamasfe/opa-rs", rev = "3cf7fea" }
-openssl = { version = "0.10" }
-openssl-sys = { version = "0.10", features = ["vendored"] }
 opentelemetry = { version = "0.19.0", features = ["rt-tokio"] }
 opentelemetry-jaeger = { version = "0.18.0", features = [
   "rt-tokio",
@@ -110,37 +109,30 @@ pin-project-lite = "0.2"
 pinvec = "0.1.0"
 pkcs8 = { version = "0.10", features = ["std", "alloc"] }
 poem = { version = "1.3.58", features = ["opentelemetry", "websocket"] }
-portpicker = "0.1.1"
-pow_of_2 = "0.1.2"
-proptest = "1.0.0"
-prost = "0.10" # common, sawtooth-protocol, sawtooth-tp: version = "0.10.0"
-prost-build = "0.10.0"
-prost-types = "0.11.2"
-protobuf = "2.27.1"
+portpicker = "0.1"
+pow_of_2 = "0.1"
+proptest = "1"
 question = "0.2.2"
 r2d2 = "0.8.9"
-rand = { version = "0.8.5", features = ["getrandom"] }
-rand_core = "0.6.3"
+rand = { version = "0.8", features = ["getrandom"] }
+rand_core = "0.6"
 rdf-types = "0.14"
-reqwest = "0.11.20"
+reqwest = "0.11"
 rust-embed = { version = "6.6.0", features = [
   "debug-embed",
   "include-exclude",
 ] }
-sawtooth-sdk = { git = "https://github.com/hyperledger/sawtooth-sdk-rust", rev = "5a300de" }
 secret-vault = { version = "1.8", features = [] }
 secret-vault-value = "0.3"
-serde = "1.0.152"
-serde_derive = "1.0.152"
-serde_json = "1.0.93"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
 serde_yaml = "0.9.14"
 shellexpand = "3.0.0"
-macro-attr-2018 = "3.0.0"
-newtype-derive-2018 = "0.2.1"
 temp-dir = "0.1.11"
 tempfile = "3.4.0"
+testcontainers = "0.14"
 thiserror = "1.0"
-tmq = "0.3"
 tokio = { version = "1.27", features = [
   "time",
   "macros",
@@ -160,10 +152,8 @@ tracing-subscriber = { version = "0.3.15", features = [
   "json",
 ] }
 tungstenite = "0.20.1"
-url = "2.3.1"
-user-error = "1.2.8"
-uuid = "1.2.2"
+url = "2"
+user-error = "1"
+uuid = "1"
 valico = "3.6.0"
-vaultrs = "*"
-zmq = { version = "0.9", features = ["vendored"] }
-testcontainers = "0.14"
+vaultrs = "0.7"

crates/api/Cargo.toml

@@ -6,6 +6,7 @@ version = "0.7.5"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = { workspace = true }
 async-graphql = { workspace = true, features = [
   "opentelemetry",
   "chrono",
@@ -14,23 +15,25 @@ async-graphql = { workspace = true, features = [
   "uuid",
 ] }
 async-graphql-poem = { workspace = true }
-async-stl-client = { workspace = true }
 async-stream = { workspace = true }
 async-trait = { workspace = true }
 base64 = { workspace = true }
 cached = { workspace = true }
 cfg-if = { workspace = true }
-chronicle-protocol = { path = "../chronicle-protocol" }
 chronicle-signing = { workspace = true }
 chronicle-telemetry = { path = "../chronicle-telemetry" }
 chrono = { workspace = true }
-common = { path = "../common" }
+common = { path = "../common", features = [
+  "json-ld",
+  "diesel-bindings",
+  "graphql-bindings",
+  "std",
+] }
 custom_error = { workspace = true }
 derivative = { workspace = true }
 diesel = { workspace = true }
 diesel_migrations = { workspace = true }
 futures = { workspace = true }
-glob = { workspace = true }
 hex = { workspace = true }
 iref = { workspace = true }
 json-ld = { workspace = true }
@@ -42,20 +45,19 @@ opa = { workspace = true }
 opentelemetry = { workspace = true }
 parking_lot = { workspace = true }
 poem = { workspace = true }
-portpicker = { workspace = true }
-prost = { workspace = true }
-protobuf = { workspace = true }
+protocol-substrate = { path = "../protocol-substrate" }
+protocol-substrate-chronicle = { path = "../protocol-substrate-chronicle" }
+protocol-substrate-opa = { path = "../protocol-substrate-opa" }
 r2d2 = { workspace = true }
 rand = { workspace = true }
 rand_core = { workspace = true }
 reqwest = { workspace = true }
-sawtooth-sdk = { workspace = true }
-sawtooth_tp = { path = "../sawtooth-tp" }
+rust-embed = { workspace = true }
 serde = { workspace = true }
 serde_derive = { workspace = true }
 serde_json = { workspace = true }
+testcontainers = { workspace = true, optional = true }
 thiserror = { workspace = true }
-tmq = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
 tracing = { workspace = true }
@@ -65,16 +67,13 @@ uuid = { workspace = true }
 
 [dev-dependencies]
 assert_fs          = { workspace = true }
-chronicle-protocol = { path = "../chronicle-protocol" }
+embedded-substrate = { path = "../embedded-substrate" }
 insta              = { workspace = true, features = ["json", "yaml"] }
-opa-tp-protocol    = { path = "../opa-tp-protocol" }
 tempfile           = { workspace = true }
 
 [build-dependencies]
 
 [features]
-devmode = ["inmem"]
+devmode = ["testcontainers"]
 
-# Use an in-memory stub ledger
-inmem  = []
 strict = []

crates/api/src/chronicle_graphql/mod.rs

@@ -12,9 +12,9 @@ use chrono::NaiveDateTime;
 use common::{
 	identity::{AuthId, IdentityError, JwtClaims, OpaData, SignedIdentity},
 	ledger::{SubmissionError, SubmissionStage},
-	opa::{ExecutorContext, OpaExecutorError},
+	opa::std::{ExecutorContext, OpaExecutorError},
 	prov::{
-		to_json_ld::ToJson, ChronicleIri, ChronicleTransactionId, ExternalId, ExternalIdPart,
+		json_ld::ToJson, ChronicleIri, ChronicleTransactionId, ExternalId, ExternalIdPart,
 		ProvModel,
 	},
 };
@@ -292,10 +292,7 @@ impl From<SignedIdentity> for CommitIdentity {
 		CommitIdentity {
 			identity: identity.identity,
 			signature: identity.signature.map(|x| hex::encode(&*x)).unwrap_or_default(),
-			verifying_key: identity
-				.verifying_key
-				.map(|x| hex::encode(x.to_bytes()))
-				.unwrap_or_default(),
+			verifying_key: identity.verifying_key.map(hex::encode).unwrap_or_default(),
 		}
 	}
 }
@@ -810,8 +807,8 @@ impl IriEndpoint {
 								.body("failed to compact JSON response"))
 						},
 					},
-					Err(StoreError::Db(diesel::result::Error::NotFound))
-					| Err(StoreError::RecordNotFound) => {
+					Err(StoreError::Db(diesel::result::Error::NotFound)) |
+					Err(StoreError::RecordNotFound) => {
 						tracing::debug!("not found: {prov_type} {} in {ns}", id.external_id_part());
 						Ok(poem::Response::builder()
 							.status(StatusCode::NOT_FOUND)
@@ -875,9 +872,8 @@ impl IriEndpoint {
 
 		match ChronicleIri::from_str(&ns_iri.iri) {
 			Ok(iri) => Ok(Ok((ns_iri.ns.into(), iri))),
-			Err(error) => {
-				Ok(Err(Response::builder().status(StatusCode::NOT_FOUND).body(error.to_string())))
-			},
+			Err(error) =>
+				Ok(Err(Response::builder().status(StatusCode::NOT_FOUND).body(error.to_string()))),
 		}
 	}
 
@@ -888,24 +884,21 @@ impl IriEndpoint {
 		claims: Option<&JwtClaims>,
 	) -> poem::Result<poem::Response> {
 		match self.parse_ns_iri_from_uri_path(req).await? {
-			Ok((ns, ChronicleIri::Activity(id))) => {
+			Ok((ns, ChronicleIri::Activity(id))) =>
 				self.response_for_query(claims, "activity", &id, &ns, |mut conn, id, ns| {
 					self.store.prov_model_for_activity_id(&mut conn, id, ns)
 				})
-				.await
-			},
-			Ok((ns, ChronicleIri::Agent(id))) => {
+				.await,
+			Ok((ns, ChronicleIri::Agent(id))) =>
 				self.response_for_query(claims, "agent", &id, &ns, |mut conn, id, ns| {
 					self.store.prov_model_for_agent_id(&mut conn, id, ns)
 				})
-				.await
-			},
-			Ok((ns, ChronicleIri::Entity(id))) => {
+				.await,
+			Ok((ns, ChronicleIri::Entity(id))) =>
 				self.response_for_query(claims, "entity", &id, &ns, |mut conn, id, ns| {
 					self.store.prov_model_for_entity_id(&mut conn, id, ns)
 				})
-				.await
-			},
+				.await,
 			Ok(_) => Ok(poem::Response::builder()
 				.status(StatusCode::NOT_FOUND)
 				.body("may query only: activity, agent, entity")),

crates/api/src/chronicle_graphql/mutation.rs

@@ -1,10 +1,9 @@
 //! Primitive mutation operations that are not in terms of particular domain types
-
+use crate::commands::{ActivityCommand, AgentCommand, ApiCommand, ApiResponse, EntityCommand};
 use async_graphql::Context;
 use chrono::{DateTime, Utc};
 use common::{
 	attributes::Attributes,
-	commands::{ActivityCommand, AgentCommand, ApiCommand, ApiResponse, EntityCommand},
 	identity::AuthId,
 	prov::{operations::DerivationType, ActivityId, AgentId, EntityId, Role},
 };

crates/api/src/commands.rs

@@ -1,4 +1,5 @@
 use core::pin::Pin;
+use std::{path::PathBuf, sync::Arc};
 #[cfg(feature = "std")]
 use std::{path::PathBuf, sync::Arc};
 
@@ -7,7 +8,7 @@ use futures::AsyncRead;
 
 use serde::{Deserialize, Serialize};
 
-use crate::{
+use common::{
 	attributes::Attributes,
 	prov::{
 		operations::{ChronicleOperation, DerivationType},
@@ -16,13 +17,6 @@ use crate::{
 	},
 };
 
-#[cfg(not(feature = "std"))]
-use parity_scale_codec::{
-	alloc::boxed::Box, alloc::string::String, alloc::sync::Arc, alloc::vec::Vec,
-};
-#[cfg(not(feature = "std"))]
-use scale_info::prelude::*;
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum NamespaceCommand {
 	Create { external_id: ExternalId },
@@ -47,7 +41,6 @@ pub enum AgentCommand {
 		role: Option<Role>,
 	},
 }
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ActivityCommand {
 	Create {
@@ -156,8 +149,7 @@ impl ActivityCommand {
 #[derive(Clone)]
 pub enum PathOrFile {
 	Path(PathBuf),
-	File(Arc<Pin<Box<dyn AsyncRead + Sync + Send>>>), /* Non serialisable variant, used in
-	                                                   * process */
+	File(Arc<Pin<Box<dyn AsyncRead + Sync + Send>>>),
 }
 
 impl core::fmt::Debug for PathOrFile {