From fe4f86970bf790f7fdabfeac7c7d9748ff2006e6 Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 12 Mar 2024 12:55:41 +0300 Subject: [PATCH] * Initial support for arrow-flight Re organise persistence into seperate crate, add arrow and schema production * Schema generation, caching etc * Service implementation for do_put * Only attributes / exists so far, no relations Signed-off-by: Ryan --- .circleci/config.yml | 6 +- Cargo.lock | 671 ++++++++++---- Cargo.toml | 10 +- crates/api/Cargo.toml | 27 +- crates/api/src/chronicle_graphql/activity.rs | 56 +- crates/api/src/chronicle_graphql/agent.rs | 42 +- .../src/chronicle_graphql/authorization.rs | 14 +- .../src/chronicle_graphql/cursor_project.rs | 37 + crates/api/src/chronicle_graphql/entity.rs | 31 +- crates/api/src/chronicle_graphql/mod.rs | 119 +-- crates/api/src/chronicle_graphql/mutation.rs | 16 +- crates/api/src/chronicle_graphql/query.rs | 23 +- crates/api/src/commands.rs | 15 +- crates/api/src/lib.rs | 362 +++++--- crates/chronicle-arrow/Cargo.toml | 44 + crates/chronicle-arrow/src/lib.rs | 865 ++++++++++++++++++ .../src/peekableflightdatastream.rs | 89 ++ crates/chronicle-persistence/Cargo.toml | 29 + .../2022-01-18-144301_ledger_sync/down.sql | 0 .../2022-01-18-144301_ledger_sync/up.sql | 0 .../2023-04-11-111125_generation/down.sql | 0 .../2023-04-11-111125_generation/up.sql | 0 .../down.sql | 0 .../2023-06-16-145625_attribution_fix/up.sql | 0 .../2023-07-12-123150_attachment/down.sql | 0 .../2023-07-12-123150_attachment/up.sql | 0 .../src/cursor.rs} | 42 +- .../src}/database.rs | 2 +- .../src/lib.rs} | 99 +- .../src}/query.rs | 8 - crates/chronicle-persistence/src/queryable.rs | 57 ++ .../src}/schema.rs | 0 crates/chronicle-signing/src/error.rs | 1 + crates/chronicle-signing/src/lib.rs | 1 + crates/chronicle-synth/Cargo.toml | 6 +- crates/chronicle-synth/src/domain.rs | 4 +- crates/chronicle-synth/src/error.rs | 24 +- .../chronicle-test-infrastructure/Cargo.toml | 2 +- .../src/api_test.rs | 26 +- .../src/substitutes/mod.rs | 32 +- .../src/substitutes/stubstrate.rs | 10 +- crates/chronicle/Cargo.toml | 28 +- crates/chronicle/src/bootstrap/cli.rs | 114 ++- crates/chronicle/src/bootstrap/mod.rs | 17 +- crates/chronicle/src/codegen/linter.rs | 10 +- crates/chronicle/src/codegen/mod.rs | 30 +- crates/chronicle/src/lib.rs | 1 + crates/common/Cargo.toml | 9 +- .../codegen/model.rs => common/src/domain.rs} | 88 +- crates/common/src/identity.rs | 12 +- crates/common/src/ledger.rs | 23 +- crates/common/src/lib.rs | 2 + crates/common/src/opa/std.rs | 48 +- crates/common/src/protocol.rs | 69 +- crates/common/src/prov/id/mod.rs | 8 + .../src/prov/model/json_ld/from_json_ld.rs | 92 +- crates/common/src/prov/model/json_ld/mod.rs | 1 + .../src/prov/model/json_ld/to_json_ld.rs | 12 +- crates/common/src/prov/model/mod.rs | 128 ++- crates/common/src/prov/model/proptest.rs | 28 +- crates/common/src/prov/operations.rs | 293 +++++- crates/common/src/prov/vocab.rs | 8 +- crates/opactl/src/main.rs | 42 +- crates/opactl/src/test/mod.rs | 67 +- crates/opactl/src/test/stubstrate.rs | 33 +- .../protocol-abstract/src/abstract_ledger.rs | 6 +- crates/protocol-substrate-opa/src/lib.rs | 10 +- .../protocol-substrate-opa/src/transaction.rs | 12 +- crates/protocol-substrate/src/subxt_client.rs | 37 +- 69 files changed, 2962 insertions(+), 1036 deletions(-) create mode 100644 crates/api/src/chronicle_graphql/cursor_project.rs create mode 100644 crates/chronicle-arrow/Cargo.toml create mode 100644 crates/chronicle-arrow/src/lib.rs create mode 100644 crates/chronicle-arrow/src/peekableflightdatastream.rs create mode 100644 crates/chronicle-persistence/Cargo.toml rename crates/{api => chronicle-persistence}/migrations/2022-01-18-144301_ledger_sync/down.sql (100%) rename crates/{api => chronicle-persistence}/migrations/2022-01-18-144301_ledger_sync/up.sql (100%) rename crates/{api => chronicle-persistence}/migrations/2023-04-11-111125_generation/down.sql (100%) rename crates/{api => chronicle-persistence}/migrations/2023-04-11-111125_generation/up.sql (100%) rename crates/{api => chronicle-persistence}/migrations/2023-06-16-145625_attribution_fix/down.sql (100%) rename crates/{api => chronicle-persistence}/migrations/2023-06-16-145625_attribution_fix/up.sql (100%) rename crates/{api => chronicle-persistence}/migrations/2023-07-12-123150_attachment/down.sql (100%) rename crates/{api => chronicle-persistence}/migrations/2023-07-12-123150_attachment/up.sql (100%) rename crates/{api/src/chronicle_graphql/cursor_query.rs => chronicle-persistence/src/cursor.rs} (62%) rename crates/{api/src/persistence => chronicle-persistence/src}/database.rs (97%) rename crates/{api/src/persistence/mod.rs => chronicle-persistence/src/lib.rs} (96%) rename crates/{api/src/persistence => chronicle-persistence/src}/query.rs (95%) create mode 100644 crates/chronicle-persistence/src/queryable.rs rename crates/{api/src/persistence => chronicle-persistence/src}/schema.rs (100%) rename crates/{chronicle/src/codegen/model.rs => common/src/domain.rs} (96%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b591c146..d346792e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,8 +15,12 @@ jobs: steps: - checkout - run: + shell: /bin/zsh name: Build release, test e2e - command: ISOLATION_ID=${CIRCLE_SHA1} gmake test + command: | + eval "$(/opt/homebrew/bin/brew shellenv)" + . "$HOME/.cargo/env" + ISOLATION_ID=${CIRCLE_SHA1} gmake test # Orchestrate jobs using workflows # See: https://circleci.com/docs/configuration-reference/#workflows diff --git a/Cargo.lock b/Cargo.lock index 3c8f18a37..d0e1202f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,11 +95,12 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b79b82693f705137f8fb9b37871d99e4f9a7df12b917eed79c3d3954830a60b" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "const-random", "getrandom 0.2.12", "once_cell", "serde", @@ -202,9 +203,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" dependencies = [ "backtrace", ] @@ -222,6 +223,7 @@ dependencies = [ "base64 0.21.7", "cached", "cfg-if", + "chronicle-persistence", "chronicle-signing", "chronicle-telemetry", "chrono", @@ -455,6 +457,117 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "arrow-array" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" +dependencies = [ + "ahash 0.8.11", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.14.3", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "base64 0.21.7", + "chrono", + "half", + "lexical-core", + "num", +] + +[[package]] +name = "arrow-data" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-flight" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7f215461ad6346f2e4cc853e377d4e076d533e1ed78d327debe83023e3601f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-schema", + "base64 0.21.7", + "bytes", + "futures", + "paste", + "prost 0.12.3", + "tokio", + "tonic 0.10.2", +] + +[[package]] +name = "arrow-ipc" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", +] + +[[package]] +name = "arrow-schema" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" + +[[package]] +name = "arrow-select" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" +dependencies = [ + "ahash 0.8.11", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + [[package]] name = "ascii_utils" version = "0.9.3" @@ -593,7 +706,7 @@ dependencies = [ "fnv", "futures-util", "handlebars", - "http 1.0.0", + "http 1.1.0", "indexmap 2.2.5", "mime", "multer", @@ -648,7 +761,7 @@ checksum = "80a8e5eb55884a7e02e770d392088b1894c08613d0a8e5ad7c0a372518a54393" dependencies = [ "async-graphql", "futures-util", - "http 1.0.0", + "http 1.1.0", "mime", "poem", "serde_json", @@ -671,9 +784,9 @@ dependencies = [ [[package]] name = "async-io" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f97ab0c5b00a7cdbe5a371b9a782ee7be1316095885c8a4ea1daf490eb0ef65" +checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" dependencies = [ "async-lock 3.3.0", "cfg-if", @@ -853,7 +966,7 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", "itoa", @@ -879,7 +992,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "mime", "rustversion", @@ -1096,9 +1209,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" dependencies = [ "arrayref", "arrayvec 0.7.4", @@ -1221,9 +1334,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.3" +version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "byte-slice-cast" @@ -1248,6 +1361,20 @@ name = "bytemuck" version = "1.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2ef034f05691a48569bd920a96c81b9d91bbad1ab5ac7c4616c1f6ef36cb79f" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] [[package]] name = "byteorder" @@ -1369,10 +1496,11 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.88" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" dependencies = [ + "jobserver", "libc", ] @@ -1450,6 +1578,8 @@ dependencies = [ "async-graphql", "async-trait", "cfg-if", + "chronicle-arrow", + "chronicle-persistence", "chronicle-signing", "chronicle-telemetry", "chrono", @@ -1496,6 +1626,34 @@ dependencies = [ "valico", ] +[[package]] +name = "chronicle-arrow" +version = "0.1.0" +dependencies = [ + "api", + "arrow-array", + "arrow-flight", + "arrow-ipc", + "arrow-schema", + "chronicle-persistence", + "chronicle-telemetry", + "chronicle-test-infrastructure", + "common", + "futures", + "insta", + "lazy_static", + "portpicker", + "prost-types 0.12.3", + "serde", + "serde_arrow", + "serde_json", + "thiserror", + "tokio", + "tonic 0.10.2", + "tracing", + "uuid 1.7.0", +] + [[package]] name = "chronicle-domain" version = "0.7.5" @@ -1535,6 +1693,26 @@ dependencies = [ "uuid 1.7.0", ] +[[package]] +name = "chronicle-persistence" +version = "0.1.0" +dependencies = [ + "async-graphql", + "async-trait", + "chrono", + "common", + "derivative", + "diesel", + "diesel_migrations", + "hex", + "protocol-substrate-chronicle", + "r2d2", + "serde_json", + "thiserror", + "tracing", + "uuid 1.7.0", +] + [[package]] name = "chronicle-signing" version = "0.1.0" @@ -1560,6 +1738,7 @@ dependencies = [ "assert_fs", "chronicle", "clap 3.2.25", + "common", "insta", "maplit", "owo-colors", @@ -1623,9 +1802,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.34" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" +checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1726,9 +1905,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.1" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" +checksum = "b230ab84b0ffdf890d5a10abdbc8b83ae1c4918275daea1ab8801f71536b2651" dependencies = [ "clap_builder", "clap_derive 4.5.0", @@ -1736,9 +1915,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.1" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", @@ -1848,7 +2027,9 @@ dependencies = [ name = "common" version = "0.7.5" dependencies = [ + "Inflector", "anyhow", + "assert_fs", "async-graphql", "async-trait", "chrono", @@ -1883,6 +2064,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "serde_yaml", "sp-core 25.0.0", "sp-std 11.0.0", "tempfile", @@ -1929,9 +2111,9 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" dependencies = [ - "prost", - "prost-types", - "tonic", + "prost 0.11.9", + "prost-types 0.11.9", + "tonic 0.9.2", "tracing-core", ] @@ -1947,13 +2129,13 @@ dependencies = [ "futures", "hdrhistogram", "humantime", - "prost-types", + "prost-types 0.11.9", "serde", "serde_json", "thread_local", "tokio", "tokio-stream", - "tonic", + "tonic 0.9.2", "tracing", "tracing-core", "tracing-subscriber 0.3.18", @@ -1967,9 +2149,9 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const-random" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", ] @@ -2194,7 +2376,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.1", + "clap 4.5.2", "criterion-plot", "futures", "is-terminal", @@ -2408,9 +2590,9 @@ checksum = "4f8a51dd197fa6ba5b4dc98a990a43cc13693c23eb0089ebb0fcc1f04152bca6" [[package]] name = "cxx" -version = "1.0.118" +version = "1.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2673ca5ae28334544ec2a6b18ebe666c42a2650abfb48abbd532ed409a44be2b" +checksum = "635179be18797d7e10edb9cd06c859580237750c7351f39ed9b298bfc17544ad" dependencies = [ "cc", "cxxbridge-flags", @@ -2420,9 +2602,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.118" +version = "1.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9df46fe0eb43066a332586114174c449a62c25689f85a08f28fdcc8e12c380b9" +checksum = "9324397d262f63ef77eb795d900c0d682a34a43ac0932bec049ed73055d52f63" dependencies = [ "cc", "codespan-reporting", @@ -2435,15 +2617,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.118" +version = "1.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "886acf875df67811c11cd015506b3392b9e1820b1627af1a6f4e93ccdfc74d11" +checksum = "a87ff7342ffaa54b7c61618e0ce2bbcf827eba6d55b923b83d82551acbbecfe5" [[package]] name = "cxxbridge-macro" -version = "1.0.118" +version = "1.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d151cc139c3080e07f448f93a1284577ab2283d2a44acd902c6fba9ec20b6de" +checksum = "70b5b86cf65fa0626d85720619d80b288013477a91a0389fa8bc716bf4903ad1" dependencies = [ "proc-macro2", "quote", @@ -2872,7 +3054,7 @@ dependencies = [ "regex", "syn 2.0.52", "termcolor", - "toml 0.8.10", + "toml 0.8.11", "walkdir", ] @@ -3199,12 +3381,13 @@ dependencies = [ [[package]] name = "expander" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f86a749cf851891866c10515ef6c299b5c69661465e9c3bbe7e07a2b77fb0f7" +checksum = "00e83c02035136f1592a47964ea60c05a50e4ed8b5892cfac197063850898d4d" dependencies = [ "blake2 0.10.6", "fs-err", + "prettier-please", "proc-macro2", "quote", "syn 2.0.52", @@ -3338,6 +3521,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flate2" version = "1.0.28" @@ -3448,7 +3641,7 @@ dependencies = [ "Inflector", "array-bytes 6.2.2", "chrono", - "clap 4.5.1", + "clap 4.5.2", "comfy-table", "frame-benchmarking", "frame-support", @@ -3944,9 +4137,9 @@ dependencies = [ [[package]] name = "ghash" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" dependencies = [ "opaque-debug 0.3.1", "polyval", @@ -3984,7 +4177,7 @@ dependencies = [ "aho-corasick", "bstr", "log", - "regex-automata 0.4.5", + "regex-automata 0.4.6", "regex-syntax 0.8.2", ] @@ -4005,7 +4198,7 @@ version = "0.7.5" dependencies = [ "anyhow", "clap 3.2.25", - "http 0.2.11", + "http 0.2.12", "rand 0.8.5", "serde_json", "tungstenite 0.20.1", @@ -4044,7 +4237,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.11", + "http 0.2.12", "indexmap 2.2.5", "slab", "tokio", @@ -4063,7 +4256,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 1.0.0", + "http 1.1.0", "indexmap 2.2.5", "slab", "tokio", @@ -4077,8 +4270,10 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" dependencies = [ + "bytemuck", "cfg-if", "crunchy", + "num-traits", ] [[package]] @@ -4125,7 +4320,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.10", + "ahash 0.8.11", ] [[package]] @@ -4134,7 +4329,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.10", + "ahash 0.8.11", "allocator-api2", "serde", ] @@ -4170,7 +4365,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http 1.0.0", + "http 1.1.0", "httpdate", "mime", "sha1", @@ -4182,7 +4377,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.0.0", + "http 1.1.0", ] [[package]] @@ -4283,9 +4478,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -4294,9 +4489,9 @@ dependencies = [ [[package]] name = "http" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -4310,7 +4505,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.11", + "http 0.2.12", "pin-project-lite 0.2.13", ] @@ -4321,18 +4516,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http 1.0.0", + "http 1.1.0", ] [[package]] name = "http-body-util" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", - "futures-util", - "http 1.0.0", + "futures-core", + "http 1.1.0", "http-body 1.0.0", "pin-project-lite 0.2.13", ] @@ -4372,7 +4567,7 @@ dependencies = [ "futures-core", "futures-util", "h2 0.3.24", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "httparse", "httpdate", @@ -4395,7 +4590,7 @@ dependencies = [ "futures-channel", "futures-util", "h2 0.4.2", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "httparse", "httpdate", @@ -4412,7 +4607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http 0.2.11", + "http 0.2.12", "hyper 0.14.28", "log", "rustls 0.21.10", @@ -4455,7 +4650,7 @@ checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "hyper 1.2.0", "pin-project-lite 0.2.13", @@ -4537,7 +4732,7 @@ dependencies = [ "ipnet", "log", "rtnetlink", - "system-configuration", + "system-configuration 0.5.1", "tokio", "windows", ] @@ -4552,7 +4747,7 @@ dependencies = [ "globset", "log", "memchr", - "regex-automata 0.4.5", + "regex-automata 0.4.6", "same-file", "walkdir", "winapi-util", @@ -4658,9 +4853,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.35.1" +version = "1.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c985c1bef99cf13c58fade470483d81a2bfe846ebde60ed28cc2dddec2df9e2" +checksum = "0a7c22c4d34ef4788c351e971c52bfdfe7ea2766f8c5466bc175dd46e52ac22e" dependencies = [ "console", "lazy_static", @@ -4775,6 +4970,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -4790,11 +4994,20 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jobserver" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -5004,7 +5217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8b3815d9f5d5de348e5f162b316dc9cdf4548305ebb15b4eb9328e66cf27d7a" dependencies = [ "futures-util", - "http 0.2.11", + "http 0.2.12", "jsonrpsee-core 0.16.3", "jsonrpsee-types 0.16.3", "pin-project", @@ -5025,7 +5238,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9f9ed46590a8d5681975f126e22531698211b926129a40a2db47cbca429220" dependencies = [ "futures-util", - "http 0.2.11", + "http 0.2.12", "jsonrpsee-core 0.21.0", "pin-project", "rustls-native-certs 0.7.0", @@ -5151,7 +5364,7 @@ checksum = "cf4d945a6008c9b03db3354fb3c83ee02d2faa9f2e755ec1dfb69c3551b8f4ba" dependencies = [ "futures-channel", "futures-util", - "http 0.2.11", + "http 0.2.12", "hyper 0.14.28", "jsonrpsee-core 0.16.3", "jsonrpsee-types 0.16.3", @@ -5198,7 +5411,7 @@ version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e1b3975ed5d73f456478681a417128597acd6a2487855fdb7b4a3d4d195bf5e" dependencies = [ - "http 0.2.11", + "http 0.2.12", "jsonrpsee-client-transport 0.16.3", "jsonrpsee-core 0.16.3", "jsonrpsee-types 0.16.3", @@ -5210,11 +5423,11 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a071f4f7efc9a9118dfb627a0a94ef247986e1ab8606a4c806ae2b3aa3b6978" dependencies = [ - "ahash 0.8.10", + "ahash 0.8.11", "anyhow", "base64 0.21.7", "bytecount", - "clap 4.5.1", + "clap 4.5.2", "fancy-regex", "fraction", "getrandom 0.2.12", @@ -5438,9 +5651,9 @@ checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2caa5afb8bf9f3a2652760ce7d4f62d21c4d5a423e68466fca30df82f2330164" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", "windows-targets 0.52.4", @@ -6268,7 +6481,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" dependencies = [ - "ahash 0.8.10", + "ahash 0.8.11", "metrics-macros", "portable-atomic", ] @@ -6361,9 +6574,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -6456,7 +6669,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http 1.0.0", + "http 1.1.0", "httparse", "log", "memchr", @@ -6716,7 +6929,7 @@ checksum = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65" name = "node-chronicle" version = "4.0.0-dev" dependencies = [ - "clap 4.5.1", + "clap 4.5.2", "frame-benchmarking", "frame-benchmarking-cli", "frame-system", @@ -6925,7 +7138,7 @@ dependencies = [ "base64 0.13.1", "chrono", "getrandom 0.2.12", - "http 0.2.11", + "http 0.2.12", "rand 0.8.5", "reqwest", "serde", @@ -7004,7 +7217,7 @@ dependencies = [ "async-trait", "chronicle-signing", "chronicle-telemetry", - "clap 4.5.1", + "clap 4.5.2", "common", "const_format", "frame-support", @@ -7121,7 +7334,7 @@ checksum = "7f51189ce8be654f9b5f7e70e49967ed894e84a06fc35c6c042e64ac1fc5399e" dependencies = [ "async-trait", "bytes", - "http 0.2.11", + "http 0.2.12", "opentelemetry", ] @@ -7524,9 +7737,9 @@ checksum = "b687ff7b5da449d39e418ad391e5e08da53ec334903ddbb921db208908fc372c" [[package]] name = "pest" -version = "2.7.7" +version = "2.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c0dcc30b6a27553f9cc242972b67f75b60eb0db71f0b5462f38b058c41546" +checksum = "56f8023d0fb78c8e03784ea1c7f3fa36e68a723138990b8d5a47d916b651e7a8" dependencies = [ "memchr", "thiserror", @@ -7535,9 +7748,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.7" +version = "2.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e1288dbd7786462961e69bfd4df7848c1e37e8b74303dbdab82c3a9cdd2809" +checksum = "b0d24f72393fd16ab6ac5738bc33cdb6a9aa73f8b902e8fe29cf4e67d7dd1026" dependencies = [ "pest", "pest_generator", @@ -7545,9 +7758,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.7" +version = "2.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1381c29a877c6d34b8c176e734f35d7f7f5b3adaefe940cb4d1bb7af94678e2e" +checksum = "fdc17e2a6c7d0a492f0158d7a4bd66cc17280308bbaff78d5bef566dca35ab80" dependencies = [ "pest", "pest_meta", @@ -7558,9 +7771,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.7" +version = "2.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0934d6907f148c22a3acbda520c7eed243ad7487a30f51f6ce52b58b7077a8a" +checksum = "934cd7631c050f4674352a6e835d5f6711ffbfb9345c2fc0107155ac495ae293" dependencies = [ "once_cell", "pest", @@ -7617,18 +7830,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", @@ -7726,16 +7939,16 @@ dependencies = [ [[package]] name = "poem" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a712ff53e257d60d3d22936c51cafa606552129d55539c8a400de44eff676d" +checksum = "ec6f0033707ce6eb76240e39a592e7eac2f702768636cb03969293227ca8e642" dependencies = [ "async-trait", "base64 0.21.7", "bytes", "futures-util", "headers", - "http 1.0.0", + "http 1.1.0", "http-body-util", "hyper 1.2.0", "hyper-util", @@ -7765,11 +7978,11 @@ dependencies = [ [[package]] name = "poem-derive" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e2bd3f82499a00ecb2d0e06bb1b9a17540aeaa28bab14336a94255ff5185f8f" +checksum = "4826d63b6760f8e5d24be9f738a5e38a43d726f32a3c2cc9388b1cc66e587f5c" dependencies = [ - "proc-macro-crate 2.0.0", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", "syn 2.0.52", @@ -7802,9 +8015,9 @@ dependencies = [ [[package]] name = "polyval" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52cff9d1d4dee5fe6d03729099f4a310a41179e0a10dbf542039873f2e826fb" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", "cpufeatures", @@ -7889,6 +8102,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "prettier-please" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22020dfcf177fcc7bf5deaf7440af371400c67c0de14c399938d8ed4fb4645d3" +dependencies = [ + "proc-macro2", + "syn 2.0.52", +] + [[package]] name = "pretty_dtoa" version = "0.3.0" @@ -7900,9 +8123,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.1.25" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +checksum = "f28f53e8b192565862cf99343194579a022eb9c7dd3a8d03134734803c7b3125" dependencies = [ "proc-macro2", "syn 1.0.109", @@ -7996,9 +8219,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -8067,7 +8290,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.11.9", +] + +[[package]] +name = "prost" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" +dependencies = [ + "bytes", + "prost-derive 0.12.3", ] [[package]] @@ -8083,9 +8316,9 @@ dependencies = [ "log", "multimap", "petgraph", - "prettyplease 0.1.25", - "prost", - "prost-types", + "prettyplease 0.1.11", + "prost 0.11.9", + "prost-types 0.11.9", "regex", "syn 1.0.109", "tempfile", @@ -8105,13 +8338,35 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "prost-types" version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ - "prost", + "prost 0.11.9", +] + +[[package]] +name = "prost-types" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" +dependencies = [ + "prost 0.12.3", ] [[package]] @@ -8208,9 +8463,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.25" +version = "2.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc7ffe15173de4c22def678345bb2ac7e1646b83a37c1484572732b5a79a49e" +checksum = "610dd51a0e8bf3e9d042b3c4fa6e66a6d0f70f9a624db7a49348c55046faba35" dependencies = [ "psl-types", ] @@ -8574,7 +8829,7 @@ checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.5", + "regex-automata 0.4.6", "regex-syntax 0.8.2", ] @@ -8589,9 +8844,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", @@ -8618,9 +8873,9 @@ checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" [[package]] name = "reqwest" -version = "0.11.24" +version = "0.11.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +checksum = "0eea5a9eb898d3783f17c6407670e3592fd174cb81a10e51d4c37f49450b9946" dependencies = [ "base64 0.21.7", "bytes", @@ -8628,7 +8883,7 @@ dependencies = [ "futures-core", "futures-util", "h2 0.3.24", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", "hyper-rustls", @@ -8647,7 +8902,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", + "system-configuration 0.6.0", "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", @@ -8911,7 +9166,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", - "http 0.2.11", + "http 0.2.12", "reqwest", "rustify_derive", "serde", @@ -9260,7 +9515,7 @@ checksum = "22c61058223f80c1f961b03f7737529609a3283eef91129e971a1966101c18ea" dependencies = [ "array-bytes 6.2.2", "chrono", - "clap 4.5.1", + "clap 4.5.2", "fdlimit", "futures", "libp2p-identity", @@ -9410,7 +9665,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30cbc5db21ea2c4ba65b23315e73e69e8155630fb47c84b93d40b0e759c9d86d" dependencies = [ - "ahash 0.8.10", + "ahash 0.8.11", "array-bytes 6.2.2", "async-trait", "dyn-clone", @@ -9639,7 +9894,7 @@ dependencies = [ "futures", "libp2p-identity", "log", - "prost", + "prost 0.11.9", "prost-build", "sc-client-api", "sc-network", @@ -9673,7 +9928,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b884a9f7cd348c4c1899c0bbf95237e39dffba4baec48d4b98c1046f6bb04fa5" dependencies = [ - "ahash 0.8.10", + "ahash 0.8.11", "futures", "futures-timer", "libp2p", @@ -9698,7 +9953,7 @@ dependencies = [ "libp2p-identity", "log", "parity-scale-codec", - "prost", + "prost 0.11.9", "prost-build", "sc-client-api", "sc-network", @@ -9724,7 +9979,7 @@ dependencies = [ "log", "mockall", "parity-scale-codec", - "prost", + "prost 0.11.9", "prost-build", "sc-client-api", "sc-consensus", @@ -9868,7 +10123,7 @@ version = "10.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8083e1b026dcf397f8c1122b3fba6cc744c6962996df6a30e0fb75223f7637" dependencies = [ - "http 0.2.11", + "http 0.2.12", "jsonrpsee 0.16.3", "log", "serde_json", @@ -10365,7 +10620,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "772575a524feeb803e5b0fcbc6dd9f367e579488197c94c6e4023aad2305774d" dependencies = [ - "ahash 0.8.10", + "ahash 0.8.11", "cfg-if", "hashbrown 0.13.2", ] @@ -10587,6 +10842,22 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_arrow" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99e774f158723ad8fe05b884fd2f89355934599a11817edd0dc854cb1ef0e42a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "bytemuck", + "chrono", + "half", + "serde", +] + [[package]] name = "serde_bytes" version = "0.11.14" @@ -10620,9 +10891,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ "itoa", "serde", @@ -11051,7 +11322,7 @@ dependencies = [ "bytes", "flate2", "futures", - "http 0.2.11", + "http 0.2.12", "httparse", "log", "rand 0.8.5", @@ -11997,7 +12268,7 @@ version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e359b358263cc322c3f678c272a3a519621d9853dcfa1374dfcbdb5f54c6f85" dependencies = [ - "ahash 0.8.10", + "ahash 0.8.11", "hash-db", "hashbrown 0.13.2", "lazy_static", @@ -12022,7 +12293,7 @@ version = "29.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e4d24d84a0beb44a71dcac1b41980e1edf7fb722c7f3046710136a283cd479b" dependencies = [ - "ahash 0.8.10", + "ahash 0.8.11", "hash-db", "lazy_static", "memory-db", @@ -12302,13 +12573,13 @@ dependencies = [ [[package]] name = "substrate-bip39" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e620c7098893ba667438b47169c00aacdd9e7c10e042250ce2b60b087ec97328" +checksum = "6a7590dc041b9bc2825e52ce5af8416c73dbe9d0654402bfd4b4941938b94d8f" dependencies = [ "hmac 0.11.0", "pbkdf2 0.8.0", - "schnorrkel 0.9.1", + "schnorrkel 0.11.4", "sha2 0.9.9", "zeroize", ] @@ -12552,7 +12823,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" +dependencies = [ + "bitflags 2.4.2", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -12565,6 +12847,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" @@ -12640,18 +12932,18 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", @@ -12942,14 +13234,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a9aad4a3066010876e8dcf5a8a06e70a558751117a145c6ce2b82c2e2054290" +checksum = "af06656561d28735e9c1cd63dfd57132c8155426aa6af24f36a00a351f88c48e" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.6", + "toml_edit 0.22.7", ] [[package]] @@ -12998,9 +13290,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.6" +version = "0.22.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1b5fd4128cc8d3e0cb74d4ed9a9cc7c7284becd4df68f5f940e1ad123606f6" +checksum = "18769cd1cec395d70860ceb4d932812a0b4d06b1a4bb336745a4d21b9496e992" dependencies = [ "indexmap 2.2.5", "serde", @@ -13022,13 +13314,40 @@ dependencies = [ "futures-core", "futures-util", "h2 0.3.24", - "http 0.2.11", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.11.9", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "h2 0.3.24", + "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", - "prost", + "prost 0.12.3", "tokio", "tokio-stream", "tower", @@ -13067,7 +13386,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "http-range-header", "pin-project-lite 0.2.13", @@ -13306,7 +13625,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "845090aa8572116b06813914fc1d09448fe895d82982b63d58de4f91b4eb79b6" dependencies = [ "async-trait", - "clap 4.5.1", + "clap 4.5.2", "frame-remote-externalities", "frame-try-runtime", "hex", @@ -13351,7 +13670,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 0.2.11", + "http 0.2.12", "httparse", "log", "rand 0.8.5", @@ -13370,7 +13689,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.0.0", + "http 1.1.0", "httparse", "log", "rand 0.8.5", @@ -13611,7 +13930,7 @@ dependencies = [ "async-trait", "bytes", "derive_builder", - "http 0.2.11", + "http 0.2.12", "reqwest", "rustify", "rustify_derive", @@ -13712,9 +14031,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -13722,9 +14041,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", @@ -13737,9 +14056,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -13749,9 +14068,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -13759,9 +14078,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", @@ -13772,9 +14091,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-instrument" @@ -14087,9 +14406,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -14150,9 +14469,9 @@ checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" [[package]] name = "wildmatch" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "495ec47bf3c1345005f40724f0269362c8556cbc43aed0526ed44cae1d35fceb" +checksum = "017f0a8ed8331210d91b7a4c30d4edef8f21a65c02f2540496e2e79725f6d8a8" [[package]] name = "winapi" diff --git a/Cargo.toml b/Cargo.toml index bdce78468..1ae53bc29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,8 @@ members = [ "crates/chronicle-synth", "crates/chronicle-signing", "crates/chronicle-telemetry", + "crates/chronicle-persistence", + "crates/chronicle-arrow", "crates/gq-subscribe", "crates/id-provider", "crates/pallet-chronicle", @@ -26,11 +28,17 @@ members = [ "crates/protocol-substrate-chronicle", "crates/protocol-substrate-opa", "crates/chronicle-test-infrastructure", + "crates/chronicle-arrow", + "crates/chronicle-persistence", ] [workspace.dependencies] Inflector = "0.11.4" anyhow = { version = "^1", features = ["backtrace"] } +arrow-array = { versipn = "^0.50" } +arrow-flight = { versipn = "^0.50" } +arrow-ipc = { versipn = "^0.50" } +arrow-schema = { versipn = "^0.50" } assert_fs = "1.0" async-graphql = "^7" async-graphql-poem = "^7" @@ -107,11 +115,11 @@ pin-project-lite = "0.2" pinvec = "0.1.0" pkcs8 = { version = "0.10", features = ["std", "alloc"] } poem = { version = "^2", features = ["opentelemetry", "websocket"] } +poem-grpc = { version = "^0.3" } portpicker = "0.1" pow_of_2 = "0.1" proptest = "1" question = "0.2.2" -r2d2 = "0.8.9" rand = { version = "0.8", features = ["getrandom"] } rand_core = "0.6" rdf-types = "0.14" diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 622b133bc..b2f7631d9 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -20,15 +20,7 @@ async-trait = { workspace = true } base64 = { workspace = true } cached = { workspace = true } cfg-if = { workspace = true } -chronicle-signing = { workspace = true } -chronicle-telemetry = { path = "../chronicle-telemetry" } chrono = { workspace = true } -common = { path = "../common", features = [ - "json-ld", - "diesel-bindings", - "graphql-bindings", - "std", -] } custom_error = { workspace = true } derivative = { workspace = true } diesel = { workspace = true } @@ -45,10 +37,6 @@ opa = { workspace = true } opentelemetry = { workspace = true } parking_lot = { workspace = true } poem = { workspace = true } -protocol-substrate = { path = "../protocol-substrate" } -protocol-substrate-chronicle = { path = "../protocol-substrate-chronicle" } -protocol-substrate-opa = { path = "../protocol-substrate-opa" } -r2d2 = { workspace = true } rand = { workspace = true } rand_core = { workspace = true } reqwest = { workspace = true } @@ -63,6 +51,21 @@ tracing = { workspace = true } url = { workspace = true } user-error = { workspace = true } uuid = { workspace = true } +#Local deps +chronicle-persistence = { path = "../chronicle-persistence" } +chronicle-signing = { workspace = true } +chronicle-telemetry = { path = "../chronicle-telemetry" } +common = { path = "../common", features = [ + "json-ld", + "diesel-bindings", + "graphql-bindings", + "std", +] } +protocol-substrate = { path = "../protocol-substrate" } +protocol-substrate-chronicle = { path = "../protocol-substrate-chronicle" } +protocol-substrate-opa = { path = "../protocol-substrate-opa" } +r2d2 = { version = "^0.8.1" } + [dev-dependencies] assert_fs = { workspace = true } diff --git a/crates/api/src/chronicle_graphql/activity.rs b/crates/api/src/chronicle_graphql/activity.rs index 5389c6989..2dbc6055e 100644 --- a/crates/api/src/chronicle_graphql/activity.rs +++ b/crates/api/src/chronicle_graphql/activity.rs @@ -1,5 +1,8 @@ -use super::{Activity, Agent, Entity, Namespace, Store}; use async_graphql::Context; +use chronicle_persistence::{ + queryable::{Activity, Agent, Entity, Namespace}, + Store, +}; use common::prov::Role; use diesel::prelude::*; use std::collections::HashMap; @@ -8,10 +11,10 @@ pub async fn namespace<'a>( namespaceid: i32, ctx: &Context<'a>, ) -> async_graphql::Result { - use crate::persistence::schema::namespace::{self, dsl}; + use chronicle_persistence::schema::namespace::{self, dsl}; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; Ok(namespace::table .filter(dsl::id.eq(namespaceid)) @@ -22,7 +25,7 @@ pub async fn was_associated_with<'a>( id: i32, ctx: &Context<'a>, ) -> async_graphql::Result, Option, Option)>> { - use crate::persistence::schema::{agent, association, delegation}; + use chronicle_persistence::schema::{agent, association, delegation}; #[derive(Queryable)] struct DelegationAgents { @@ -32,7 +35,7 @@ pub async fn was_associated_with<'a>( } let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; let delegation_entries = delegation::table .filter(delegation::dsl::activity_id.eq(id)) @@ -62,8 +65,8 @@ pub async fn was_associated_with<'a>( let res = association::table .filter(association::dsl::activity_id.eq(id)) - .inner_join(crate::persistence::schema::agent::table) - .order(crate::persistence::schema::agent::external_id) + .inner_join(chronicle_persistence::schema::agent::table) + .order(chronicle_persistence::schema::agent::external_id) .select((Agent::as_select(), association::role)) .load::<(Agent, Role)>(&mut connection)? .into_iter() @@ -89,16 +92,16 @@ pub async fn was_associated_with<'a>( } pub async fn used<'a>(id: i32, ctx: &Context<'a>) -> async_graphql::Result> { - use crate::persistence::schema::usage::{self, dsl}; + use chronicle_persistence::schema::usage::{self, dsl}; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; let res = usage::table .filter(dsl::activity_id.eq(id)) - .inner_join(crate::persistence::schema::entity::table) - .order(crate::persistence::schema::entity::external_id) + .inner_join(chronicle_persistence::schema::entity::table) + .order(chronicle_persistence::schema::entity::external_id) .select(Entity::as_select()) .load::(&mut connection)?; @@ -109,35 +112,34 @@ pub async fn was_informed_by<'a>( id: i32, ctx: &Context<'a>, ) -> async_graphql::Result> { - use crate::persistence::schema::wasinformedby::{self, dsl}; + use chronicle_persistence::schema::wasinformedby::{self, dsl}; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; - let res = - wasinformedby::table - .filter(dsl::activity_id.eq(id)) - .inner_join(crate::persistence::schema::activity::table.on( - wasinformedby::informing_activity_id.eq(crate::persistence::schema::activity::id), - )) - .order(crate::persistence::schema::activity::external_id) - .select(Activity::as_select()) - .load::(&mut connection)?; + let res = wasinformedby::table + .filter(dsl::activity_id.eq(id)) + .inner_join(chronicle_persistence::schema::activity::table.on( + wasinformedby::informing_activity_id.eq(chronicle_persistence::schema::activity::id), + )) + .order(chronicle_persistence::schema::activity::external_id) + .select(Activity::as_select()) + .load::(&mut connection)?; Ok(res) } pub async fn generated<'a>(id: i32, ctx: &Context<'a>) -> async_graphql::Result> { - use crate::persistence::schema::generation::{self, dsl}; + use chronicle_persistence::schema::generation::{self, dsl}; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; let res = generation::table .filter(dsl::activity_id.eq(id)) - .inner_join(crate::persistence::schema::entity::table) + .inner_join(chronicle_persistence::schema::entity::table) .select(Entity::as_select()) .load::(&mut connection)?; @@ -149,11 +151,11 @@ pub async fn load_attribute<'a>( external_id: &str, ctx: &Context<'a>, ) -> async_graphql::Result> { - use crate::persistence::schema::activity_attribute; + use chronicle_persistence::schema::activity_attribute; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; Ok(activity_attribute::table .filter( diff --git a/crates/api/src/chronicle_graphql/agent.rs b/crates/api/src/chronicle_graphql/agent.rs index a7507e34f..62a7758cf 100644 --- a/crates/api/src/chronicle_graphql/agent.rs +++ b/crates/api/src/chronicle_graphql/agent.rs @@ -1,7 +1,8 @@ -use crate::chronicle_graphql::Entity; - -use super::{Agent, Identity, Namespace, Store}; use async_graphql::Context; +use chronicle_persistence::{ + queryable::{Agent, Entity, Namespace}, + Store, +}; use common::prov::Role; use diesel::prelude::*; @@ -9,47 +10,28 @@ pub async fn namespace<'a>( namespace_id: i32, ctx: &Context<'a>, ) -> async_graphql::Result { - use crate::persistence::schema::namespace::{self, dsl}; + use chronicle_persistence::schema::namespace::{self, dsl}; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; Ok(namespace::table .filter(dsl::id.eq(namespace_id)) .first::(&mut connection)?) } -pub async fn identity<'a>( - identity_id: Option, - ctx: &Context<'a>, -) -> async_graphql::Result> { - use crate::persistence::schema::identity::{self, dsl}; - let store = ctx.data_unchecked::(); - - let mut connection = store.pool.get()?; - - if let Some(identity_id) = identity_id { - Ok(identity::table - .filter(dsl::id.eq(identity_id)) - .first::(&mut connection) - .optional()?) - } else { - Ok(None) - } -} - pub async fn acted_on_behalf_of<'a>( id: i32, ctx: &Context<'a>, ) -> async_graphql::Result)>> { - use crate::persistence::schema::{ + use chronicle_persistence::schema::{ agent as agentdsl, delegation::{self, dsl}, }; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; Ok(delegation::table .filter(dsl::delegate_id.eq(id)) @@ -68,14 +50,14 @@ pub async fn attribution<'a>( id: i32, ctx: &Context<'a>, ) -> async_graphql::Result)>> { - use crate::persistence::schema::{ + use chronicle_persistence::schema::{ attribution::{self, dsl}, entity as entity_dsl, }; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; Ok(attribution::table .filter(dsl::agent_id.eq(id)) @@ -93,11 +75,11 @@ pub async fn load_attribute<'a>( external_id: &str, ctx: &Context<'a>, ) -> async_graphql::Result> { - use crate::persistence::schema::agent_attribute; + use chronicle_persistence::schema::agent_attribute; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; Ok(agent_attribute::table .filter(agent_attribute::agent_id.eq(id).and(agent_attribute::typename.eq(external_id))) diff --git a/crates/api/src/chronicle_graphql/authorization.rs b/crates/api/src/chronicle_graphql/authorization.rs index 58e4ba997..dfe5c69b6 100644 --- a/crates/api/src/chronicle_graphql/authorization.rs +++ b/crates/api/src/chronicle_graphql/authorization.rs @@ -15,21 +15,25 @@ pub enum Error { #[error("Base64 decoding failure: {0}", source)] Base64 { #[from] + #[source] source: base64::DecodeError, }, #[error("JSON decoding failure: {0}", source)] Json { #[from] + #[source] source: serde_json::Error, }, #[error("JWT validation failure: {0}", source)] Jwks { #[from] + #[source] source: jwtk::Error, }, #[error("web access failure: {0}", source)] Reqwest { #[from] + #[source] source: reqwest::Error, }, #[error("formatting error: {0}", message)] @@ -76,7 +80,7 @@ impl TokenChecker { // should respond with JSON web key set if !status.is_success() { tracing::warn!("{uri:?} returns {status}"); - return Err(Error::UnexpectedResponse { server: format!("{uri:?}"), status }) + return Err(Error::UnexpectedResponse { server: format!("{uri:?}"), status }); } } if let Some(uri) = &self.userinfo_uri { @@ -84,7 +88,7 @@ impl TokenChecker { // should require an authorization token if !status.is_client_error() || status == StatusCode::NOT_FOUND { tracing::warn!("{uri:?} without authorization token returns {status}"); - return Err(Error::UnexpectedResponse { server: format!("{uri:?}"), status }) + return Err(Error::UnexpectedResponse { server: format!("{uri:?}"), status }); } } Ok(()) @@ -98,7 +102,7 @@ impl TokenChecker { if let Some(verifier) = &self.verifier { verifier.verify::>(token).await?; } else { - return Err(Error::Format { message: "no JWKS endpoint configured".to_string() }) + return Err(Error::Format { message: "no JWKS endpoint configured".to_string() }); } // JWT is composed of three base64-encoded components @@ -107,7 +111,7 @@ impl TokenChecker { .map(|component| BASE64_ENGINE.decode(component)) .collect::>, base64::DecodeError>>()?; if components.len() != 3 { - return Err(Error::Format { message: format!("JWT has unexpected format: {token}") }) + return Err(Error::Format { message: format!("JWT has unexpected format: {token}") }); }; if let Value::Object(claims) = serde_json::from_slice(components[1].as_slice())? { @@ -133,7 +137,7 @@ impl TokenChecker { }, _ => (), } - return Err(Error::Jwks { source }) // abort on JWKS verifier failure + return Err(Error::Jwks { source }); // abort on JWKS verifier failure }, Err(err) => error = Some(err), // could tolerate error from what may be opaque token }; diff --git a/crates/api/src/chronicle_graphql/cursor_project.rs b/crates/api/src/chronicle_graphql/cursor_project.rs new file mode 100644 index 000000000..48380e4bb --- /dev/null +++ b/crates/api/src/chronicle_graphql/cursor_project.rs @@ -0,0 +1,37 @@ +use async_graphql::{ + connection::{Edge, EmptyFields}, + OutputType, +}; +use diesel::{ + prelude::*, + r2d2::{ConnectionManager, PooledConnection}, +}; + +type Conn = PooledConnection>; + +pub fn project_to_nodes( + rx: I, + start: i64, + limit: i64, +) -> async_graphql::connection::Connection +where + T: OutputType, + I: IntoIterator, +{ + let rx = Vec::from_iter(rx); + let mut gql = async_graphql::connection::Connection::new( + rx.first().map(|(_, _total)| start > 0).unwrap_or(false), + rx.first().map(|(_, total)| start + limit < *total).unwrap_or(false), + ); + + gql.edges.append( + &mut rx + .into_iter() + .enumerate() + .map(|(pos, (agent, _count))| { + Edge::with_additional_fields((pos as i32) + (start as i32), agent, EmptyFields) + }) + .collect(), + ); + gql +} diff --git a/crates/api/src/chronicle_graphql/entity.rs b/crates/api/src/chronicle_graphql/entity.rs index 317275c0d..b10a5bc8f 100644 --- a/crates/api/src/chronicle_graphql/entity.rs +++ b/crates/api/src/chronicle_graphql/entity.rs @@ -1,5 +1,8 @@ -use super::{Activity, Agent, Entity, Namespace, Store}; use async_graphql::Context; +use chronicle_persistence::{ + queryable::{Activity, Agent, Entity, Namespace}, + Store, +}; use common::prov::{operations::DerivationType, Role}; use diesel::prelude::*; @@ -8,14 +11,14 @@ async fn typed_derivation<'a>( ctx: &Context<'a>, typ: DerivationType, ) -> async_graphql::Result> { - use crate::persistence::schema::{ + use chronicle_persistence::schema::{ derivation::{self, dsl}, entity as entitydsl, }; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; let res = derivation::table .filter(dsl::generated_entity_id.eq(id).and(dsl::typ.eq(typ))) @@ -30,11 +33,11 @@ pub async fn namespace<'a>( namespace_id: i32, ctx: &Context<'a>, ) -> async_graphql::Result { - use crate::persistence::schema::namespace::{self, dsl}; + use chronicle_persistence::schema::namespace::{self, dsl}; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; Ok(namespace::table .filter(dsl::id.eq(namespace_id)) @@ -47,10 +50,10 @@ pub async fn was_attributed_to<'a>( id: i32, ctx: &Context<'a>, ) -> async_graphql::Result)>> { - use crate::persistence::schema::{agent, attribution}; + use chronicle_persistence::schema::{agent, attribution}; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; let res = attribution::table .filter(attribution::dsl::entity_id.eq(id)) @@ -72,15 +75,15 @@ pub async fn was_generated_by<'a>( id: i32, ctx: &Context<'a>, ) -> async_graphql::Result> { - use crate::persistence::schema::generation::{self, dsl}; + use chronicle_persistence::schema::generation::{self, dsl}; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; let res = generation::table .filter(dsl::generated_entity_id.eq(id)) - .inner_join(crate::persistence::schema::activity::table) + .inner_join(chronicle_persistence::schema::activity::table) .select(Activity::as_select()) .load::(&mut connection)?; @@ -91,14 +94,14 @@ pub async fn was_derived_from<'a>( id: i32, ctx: &Context<'a>, ) -> async_graphql::Result> { - use crate::persistence::schema::{ + use chronicle_persistence::schema::{ derivation::{self, dsl}, entity as entitydsl, }; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; let res = derivation::table .filter(dsl::generated_entity_id.eq(id)) @@ -128,11 +131,11 @@ pub async fn load_attribute<'a>( external_id: &str, ctx: &Context<'a>, ) -> async_graphql::Result> { - use crate::persistence::schema::entity_attribute; + use chronicle_persistence::schema::entity_attribute; let store = ctx.data_unchecked::(); - let mut connection = store.pool.get()?; + let mut connection = store.connection()?; Ok(entity_attribute::table .filter( diff --git a/crates/api/src/chronicle_graphql/mod.rs b/crates/api/src/chronicle_graphql/mod.rs index 83658fe2a..135686aa5 100644 --- a/crates/api/src/chronicle_graphql/mod.rs +++ b/crates/api/src/chronicle_graphql/mod.rs @@ -1,14 +1,14 @@ use async_graphql::{ extensions::OpenTelemetry, http::{playground_source, GraphQLPlaygroundConfig, ALL_WEBSOCKET_PROTOCOLS}, - scalar, Context, Enum, Error, ErrorExtensions, Object, ObjectType, Schema, ServerError, - SimpleObject, Subscription, SubscriptionType, + scalar, Context, Enum, Error, ErrorExtensions, ObjectType, Schema, ServerError, SimpleObject, + Subscription, SubscriptionType, }; use async_graphql_poem::{ GraphQL, GraphQLBatchRequest, GraphQLBatchResponse, GraphQLProtocol, GraphQLSubscription, GraphQLWebSocket, }; -use chrono::NaiveDateTime; + use common::{ identity::{AuthId, IdentityError, JwtClaims, OpaData, SignedIdentity}, ledger::{SubmissionError, SubmissionStage}, @@ -21,7 +21,7 @@ use common::{ use derivative::*; use diesel::{ prelude::*, - r2d2::{ConnectionManager, Pool}, + r2d2::{ConnectionManager, Pool, PooledConnection}, PgConnection, Queryable, }; use futures::Stream; @@ -37,7 +37,6 @@ use poem::{ }, Endpoint, IntoResponse, Route, Server, }; -use r2d2::PooledConnection; use serde::{Deserialize, Serialize}; use serde_json::json; use std::{ @@ -59,87 +58,13 @@ use crate::{ApiDispatch, ApiError, StoreError}; pub mod activity; pub mod agent; mod authorization; -mod cursor_query; +mod cursor_project; pub mod entity; pub mod mutation; pub mod query; pub type AuthorizationError = authorization::Error; -#[derive(Default, Queryable, Selectable, SimpleObject)] -#[diesel(table_name = crate::persistence::schema::agent)] -pub struct Agent { - pub id: i32, - pub external_id: String, - pub namespace_id: i32, - pub domaintype: Option, - pub current: i32, - pub identity_id: Option, -} - -#[derive(Default, Queryable, Selectable)] -#[diesel(table_name = crate::persistence::schema::identity)] -pub struct Identity { - pub id: i32, - pub namespace_id: i32, - pub public_key: String, -} - -#[Object] -/// # `chronicle:Identity` -/// -/// Represents a cryptographic identity for an agent, supporting a single current -/// signing identity via `chronicle:hasIdentity` and historical identities via -/// `chronicle:hadIdentity`. -impl Identity { - async fn public_key(&self) -> &str { - &self.public_key - } -} - -#[derive(Default, Queryable, Selectable, SimpleObject)] -#[diesel(table_name = crate::persistence::schema::activity)] -pub struct Activity { - pub id: i32, - pub external_id: String, - pub namespace_id: i32, - pub domaintype: Option, - pub started: Option, - pub ended: Option, -} - -#[derive(Queryable, Selectable, SimpleObject)] -#[diesel(table_name = crate::persistence::schema::entity)] -pub struct Entity { - pub id: i32, - pub external_id: String, - pub namespace_id: i32, - pub domaintype: Option, -} - -#[derive(Default, Queryable)] -pub struct Namespace { - _id: i32, - uuid: String, - external_id: String, -} - -#[Object] -/// # `chronicle:Namespace` -/// -/// An IRI containing an external id and uuid part, used for disambiguation. -/// In order to work on the same namespace discrete Chronicle instances must share -/// the uuid part. -impl Namespace { - async fn external_id(&self) -> &str { - &self.external_id - } - - async fn uuid(&self) -> &str { - &self.uuid - } -} - #[derive(Queryable, SimpleObject)] /// # `Submission` /// @@ -199,19 +124,39 @@ pub enum TimelineOrder { #[derive(Error, Debug)] pub enum GraphQlError { #[error("Database operation failed: {0}")] - Db(#[from] diesel::result::Error), + Db( + #[from] + #[source] + diesel::result::Error, + ), #[error("Connection pool error: {0}")] - R2d2(#[from] r2d2::Error), + R2d2( + #[from] + #[source] + diesel::r2d2::Error, + ), #[error("Database connection failed: {0}")] - DbConnection(#[from] diesel::ConnectionError), + DbConnection( + #[from] + #[source] + diesel::ConnectionError, + ), #[error("API: {0}")] - Api(#[from] crate::ApiError), + Api( + #[from] + #[source] + crate::ApiError, + ), #[error("I/O: {0}")] - Io(#[from] std::io::Error), + Io( + #[from] + #[source] + std::io::Error, + ), } impl GraphQlError { @@ -755,7 +700,7 @@ where struct IriEndpoint { secconf: Option, - store: super::persistence::Store, + store: chronicle_persistence::Store, opa_executor: ExecutorContext, claim_parser: Option, } @@ -1077,7 +1022,7 @@ where let iri_endpoint = |secconf| IriEndpoint { secconf, - store: super::persistence::Store::new(pool.clone()).unwrap(), + store: chronicle_persistence::Store::new(pool.clone()).unwrap(), opa_executor: sec.opa.clone(), claim_parser: claim_parser.clone(), }; diff --git a/crates/api/src/chronicle_graphql/mutation.rs b/crates/api/src/chronicle_graphql/mutation.rs index e96255ff9..251b3aa65 100644 --- a/crates/api/src/chronicle_graphql/mutation.rs +++ b/crates/api/src/chronicle_graphql/mutation.rs @@ -16,10 +16,12 @@ async fn transaction_context<'a>( _ctx: &Context<'a>, ) -> async_graphql::Result { match res { - ApiResponse::Submission { subject, tx_id, .. } => - Ok(Submission::from_submission(&subject, &tx_id)), - ApiResponse::AlreadyRecorded { subject, .. } => - Ok(Submission::from_already_recorded(&subject)), + ApiResponse::Submission { subject, tx_id, .. } => { + Ok(Submission::from_submission(&subject, &tx_id)) + }, + ApiResponse::AlreadyRecorded { subject, .. } => { + Ok(Submission::from_already_recorded(&subject)) + }, _ => unreachable!(), } } @@ -68,7 +70,7 @@ pub async fn agent<'a>( let res = api .dispatch( ApiCommand::Agent(AgentCommand::Create { - external_id: external_id.into(), + id: external_id.into(), namespace: namespace.into(), attributes, }), @@ -94,7 +96,7 @@ pub async fn activity<'a>( let res = api .dispatch( ApiCommand::Activity(ActivityCommand::Create { - external_id: external_id.into(), + id: external_id.into(), namespace: namespace.into(), attributes, }), @@ -120,7 +122,7 @@ pub async fn entity<'a>( let res = api .dispatch( ApiCommand::Entity(EntityCommand::Create { - external_id: external_id.into(), + id: external_id.into(), namespace: namespace.into(), attributes, }), diff --git a/crates/api/src/chronicle_graphql/query.rs b/crates/api/src/chronicle_graphql/query.rs index e98a302e6..e8b97058c 100644 --- a/crates/api/src/chronicle_graphql/query.rs +++ b/crates/api/src/chronicle_graphql/query.rs @@ -6,11 +6,12 @@ use chrono::{DateTime, NaiveDate, TimeZone, Utc}; use diesel::{debug_query, pg::Pg, prelude::*}; use tracing::{debug, instrument}; -use super::{ - cursor_query::{project_to_nodes, Cursorize}, - Activity, Agent, Entity, GraphQlError, Store, TimelineOrder, +use super::{cursor_project::project_to_nodes, GraphQlError, Store, TimelineOrder}; +use chronicle_persistence::{ + cursor::Cursorize, + queryable::{Activity, Agent, Entity}, + schema::generation, }; -use crate::persistence::schema::generation; use common::prov::{ActivityId, AgentId, DomaintypeId, EntityId, ExternalIdPart}; #[allow(clippy::too_many_arguments)] @@ -29,7 +30,7 @@ pub async fn activity_timeline<'a>( first: Option, last: Option, ) -> async_graphql::Result> { - use crate::persistence::schema::{ + use chronicle_persistence::schema::{ activity, agent, association, delegation, entity, namespace::dsl as nsdsl, usage, wasinformedby, }; @@ -136,7 +137,7 @@ pub async fn entities_by_type<'a>( first: Option, last: Option, ) -> async_graphql::Result> { - use crate::persistence::schema::{entity, namespace::dsl as nsdsl}; + use chronicle_persistence::schema::{entity, namespace::dsl as nsdsl}; let store = ctx.data_unchecked::(); @@ -177,7 +178,7 @@ pub async fn activities_by_type<'a>( first: Option, last: Option, ) -> async_graphql::Result> { - use crate::persistence::schema::{activity, namespace::dsl as nsdsl}; + use chronicle_persistence::schema::{activity, namespace::dsl as nsdsl}; let store = ctx.data_unchecked::(); @@ -217,7 +218,7 @@ pub async fn agents_by_type<'a>( first: Option, last: Option, ) -> async_graphql::Result> { - use crate::persistence::schema::{agent, namespace::dsl as nsdsl}; + use chronicle_persistence::schema::{agent, namespace::dsl as nsdsl}; let store = ctx.data_unchecked::(); @@ -253,7 +254,7 @@ pub async fn agent_by_id<'a>( id: AgentId, namespace: Option, ) -> async_graphql::Result> { - use crate::persistence::schema::{ + use chronicle_persistence::schema::{ agent::{self, dsl}, namespace::dsl as nsdsl, }; @@ -276,7 +277,7 @@ pub async fn activity_by_id<'a>( id: ActivityId, namespace: Option, ) -> async_graphql::Result> { - use crate::persistence::schema::{ + use chronicle_persistence::schema::{ activity::{self, dsl}, namespace::dsl as nsdsl, }; @@ -299,7 +300,7 @@ pub async fn entity_by_id<'a>( id: EntityId, namespace: Option, ) -> async_graphql::Result> { - use crate::persistence::schema::{ + use chronicle_persistence::schema::{ entity::{self, dsl}, namespace::dsl as nsdsl, }; diff --git a/crates/api/src/commands.rs b/crates/api/src/commands.rs index baf77e4f6..b3c25c04e 100644 --- a/crates/api/src/commands.rs +++ b/crates/api/src/commands.rs @@ -19,13 +19,13 @@ use common::{ #[derive(Debug, Clone, Serialize, Deserialize)] pub enum NamespaceCommand { - Create { external_id: ExternalId }, + Create { id: ExternalId }, } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum AgentCommand { Create { - external_id: ExternalId, + id: ExternalId, namespace: ExternalId, attributes: Attributes, }, @@ -44,7 +44,7 @@ pub enum AgentCommand { #[derive(Debug, Clone, Serialize, Deserialize)] pub enum ActivityCommand { Create { - external_id: ExternalId, + id: ExternalId, namespace: ExternalId, attributes: Attributes, }, @@ -96,7 +96,7 @@ impl ActivityCommand { attributes: Attributes, ) -> Self { Self::Create { - external_id: external_id.as_ref().into(), + id: external_id.as_ref().into(), namespace: namespace.as_ref().into(), attributes, } @@ -190,7 +190,7 @@ impl<'de> Deserialize<'de> for PathOrFile { #[derive(Debug, Clone, Serialize, Deserialize)] pub enum EntityCommand { Create { - external_id: ExternalId, + id: ExternalId, namespace: ExternalId, attributes: Attributes, }, @@ -216,7 +216,7 @@ impl EntityCommand { attributes: Attributes, ) -> Self { Self::Create { - external_id: external_id.as_ref().into(), + id: external_id.as_ref().into(), namespace: namespace.as_ref().into(), attributes, } @@ -245,7 +245,6 @@ pub struct DepthChargeCommand { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ImportCommand { - pub namespace: NamespaceId, pub operations: Vec, } @@ -266,6 +265,8 @@ pub enum ApiResponse { Unit, /// The operation will not result in any data changes AlreadyRecorded { subject: ChronicleIri, prov: Box }, + /// The aggregate operation will not result in any data changes + AlreadyRecordedAll, /// The api has validated the command and submitted a transaction to a ledger Submission { subject: ChronicleIri, prov: Box, tx_id: ChronicleTransactionId }, /// The api has successfully executed the query diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index f87c07096..d70012a2e 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -1,7 +1,6 @@ #![cfg_attr(feature = "strict", deny(warnings))] pub mod chronicle_graphql; pub mod commands; -mod persistence; use chronicle_signing::{ChronicleKnownKeyNamesSigner, ChronicleSigning, SecretError}; use chrono::{DateTime, Utc}; @@ -28,16 +27,16 @@ use diesel::{r2d2::ConnectionManager, PgConnection}; use diesel_migrations::MigrationHarness; use futures::{select, FutureExt, StreamExt}; +pub use chronicle_persistence::StoreError; +use chronicle_persistence::{Store, MIGRATIONS}; +use diesel::r2d2::Pool; use metrics::histogram; use metrics_exporter_prometheus::PrometheusBuilder; -pub use persistence::StoreError; -use persistence::{Store, MIGRATIONS}; use protocol_substrate::SubxtClientError; use protocol_substrate_chronicle::{ protocol::{BlockId, FromBlock, LedgerReader, LedgerWriter}, ChronicleEvent, ChronicleTransaction, }; -use r2d2::Pool; use std::{ convert::Infallible, marker::PhantomData, @@ -52,7 +51,6 @@ use tokio::{ use commands::*; pub mod import; -pub use persistence::{database, ConnectionOptions}; use tracing::{debug, error, info, info_span, instrument, trace, warn, Instrument}; use user_error::UFE; @@ -60,19 +58,35 @@ use uuid::Uuid; #[derive(Error, Debug)] pub enum ApiError { #[error("Storage: {0:?}")] - Store(#[from] persistence::StoreError), + Store( + #[from] + #[source] + chronicle_persistence::StoreError, + ), #[error("Transaction failed: {0}")] - Transaction(#[from] diesel::result::Error), + Transaction( + #[from] + #[source] + diesel::result::Error, + ), #[error("Invalid IRI: {0}")] - Iri(#[from] iref::Error), + Iri( + #[from] + #[source] + iref::Error, + ), #[error("JSON-LD processing: {0}")] JsonLD(String), #[error("Signing: {0}")] - Signing(#[from] SecretError), + Signing( + #[from] + #[source] + SecretError, + ), #[error("No agent is currently in use, please call agent use or supply an agent in your call")] NoCurrentAgent, @@ -81,40 +95,84 @@ pub enum ApiError { ApiShutdownRx, #[error("Api shut down before send: {0}")] - ApiShutdownTx(#[from] SendError), + ApiShutdownTx( + #[from] + #[source] + SendError, + ), #[error("Ledger shut down before send: {0}")] - LedgerShutdownTx(#[from] SendError), + LedgerShutdownTx( + #[from] + #[source] + SendError, + ), #[error("Invalid socket address: {0}")] - AddressParse(#[from] AddrParseError), + AddressParse( + #[from] + #[source] + AddrParseError, + ), #[error("Connection pool: {0}")] - ConnectionPool(#[from] r2d2::Error), + ConnectionPool( + #[from] + #[source] + r2d2::Error, + ), #[error("IO error: {0}")] - InputOutput(#[from] std::io::Error), + InputOutput( + #[from] + #[source] + std::io::Error, + ), #[error("Blocking thread pool: {0}")] - Join(#[from] JoinError), + Join( + #[from] + #[source] + JoinError, + ), #[error("No appropriate activity to end")] NotCurrentActivity, #[error("Processor: {0}")] - ProcessorError(#[from] ProcessorError), + ProcessorError( + #[from] + #[source] + ProcessorError, + ), #[error("Identity: {0}")] - IdentityError(#[from] IdentityError), + IdentityError( + #[from] + #[source] + IdentityError, + ), #[error("Authentication endpoint error: {0}")] - AuthenticationEndpoint(#[from] chronicle_graphql::AuthorizationError), + AuthenticationEndpoint( + #[from] + #[source] + chronicle_graphql::AuthorizationError, + ), #[error("Substrate : {0}")] - ClientError(#[from] SubxtClientError), + ClientError( + #[from] + #[source] + SubxtClientError, + ), #[error("Submission : {0}")] - Submission(#[from] SubmissionError), + Submission( + #[from] + #[source] + SubmissionError, + ), #[error("Contradiction: {0}")] Contradiction(Contradiction), @@ -188,7 +246,7 @@ pub struct Api< submit_tx: tokio::sync::broadcast::Sender, signing: ChronicleSigning, ledger_writer: W, - store: persistence::Store, + store: chronicle_persistence::Store, uuid_source: PhantomData, } @@ -223,20 +281,18 @@ impl ApiDispatch { pub async fn handle_import_command( &self, identity: AuthId, - namespace: NamespaceId, operations: Vec, ) -> Result { - self.import_operations(identity, namespace, operations).await + self.import_operations(identity, operations).await } #[instrument] async fn import_operations( &self, identity: AuthId, - namespace: NamespaceId, operations: Vec, ) -> Result { - self.dispatch(ApiCommand::Import(ImportCommand { namespace, operations }), identity.clone()) + self.dispatch(ApiCommand::Import(ImportCommand { operations }), identity.clone()) .await } @@ -447,11 +503,12 @@ where }; match stage { - SubmissionStage::Submitted(Ok(id)) => + SubmissionStage::Submitted(Ok(id)) => { if id == tx_id { debug!("Depth charge operation submitted: {id}"); continue; - }, + } + }, SubmissionStage::Submitted(Err(err)) => { if err.tx_id() == &tx_id { error!("Depth charge transaction rejected by Chronicle: {} {}", @@ -568,31 +625,30 @@ where model.namespace_context(&namespace); model }, - ChronicleOperation::AgentExists(AgentExists { ref namespace, ref external_id }) => + ChronicleOperation::AgentExists(AgentExists { ref namespace, ref id }) => { self.store.apply_prov_model_for_agent_id( connection, model, - &AgentId::from_external_id(external_id), + id, namespace.external_id_part(), - )?, - ChronicleOperation::ActivityExists(ActivityExists { - ref namespace, - ref external_id, - }) => self.store.apply_prov_model_for_activity_id( - connection, - model, - &ActivityId::from_external_id(external_id), - namespace.external_id_part(), - )?, - ChronicleOperation::EntityExists(EntityExists { - ref namespace, - ref external_id, - }) => self.store.apply_prov_model_for_entity_id( - connection, - model, - &EntityId::from_external_id(external_id), - namespace.external_id_part(), - )?, + )? + }, + ChronicleOperation::ActivityExists(ActivityExists { ref namespace, ref id }) => { + self.store.apply_prov_model_for_activity_id( + connection, + model, + id, + namespace.external_id_part(), + )? + }, + ChronicleOperation::EntityExists(EntityExists { ref namespace, ref id }) => { + self.store.apply_prov_model_for_entity_id( + connection, + model, + id, + namespace.external_id_part(), + )? + }, ChronicleOperation::ActivityUses(ActivityUses { ref namespace, ref id, @@ -605,42 +661,47 @@ where namespace.external_id_part(), )?, ChronicleOperation::SetAttributes(ref o) => match o { - SetAttributes::Activity { namespace, id, .. } => + SetAttributes::Activity { namespace, id, .. } => { self.store.apply_prov_model_for_activity_id( connection, model, id, namespace.external_id_part(), - )?, - SetAttributes::Agent { namespace, id, .. } => + )? + }, + SetAttributes::Agent { namespace, id, .. } => { self.store.apply_prov_model_for_agent_id( connection, model, id, namespace.external_id_part(), - )?, - SetAttributes::Entity { namespace, id, .. } => + )? + }, + SetAttributes::Entity { namespace, id, .. } => { self.store.apply_prov_model_for_entity_id( connection, model, id, namespace.external_id_part(), - )?, + )? + }, }, - ChronicleOperation::StartActivity(StartActivity { namespace, id, .. }) => + ChronicleOperation::StartActivity(StartActivity { namespace, id, .. }) => { self.store.apply_prov_model_for_activity_id( connection, model, id, namespace.external_id_part(), - )?, - ChronicleOperation::EndActivity(EndActivity { namespace, id, .. }) => + )? + }, + ChronicleOperation::EndActivity(EndActivity { namespace, id, .. }) => { self.store.apply_prov_model_for_activity_id( connection, model, id, namespace.external_id_part(), - )?, + )? + }, ChronicleOperation::WasInformedBy(WasInformedBy { namespace, activity, @@ -824,18 +885,21 @@ where fn ensure_namespace( &mut self, connection: &mut PgConnection, - external_id: &ExternalId, + id: &ExternalId, ) -> Result<(NamespaceId, Vec), ApiError> { - let ns = self.store.namespace_by_external_id(connection, external_id); - - if ns.is_err() { - debug!(?ns, "Namespace does not exist, creating"); - - let uuid = U::uuid(); - let id: NamespaceId = NamespaceId::from_external_id(external_id, uuid); - Ok((id.clone(), vec![ChronicleOperation::CreateNamespace(CreateNamespace::new(id))])) - } else { - Ok((ns?.0, vec![])) + match self.store.namespace_by_external_id(connection, id) { + Ok((namespace_id, _)) => { + trace!(%id, "Namespace already exists."); + Ok((namespace_id, vec![])) + }, + Err(e) => { + debug!(error = %e, %id, "Namespace does not exist, creating."); + let uuid = Uuid::new_v4(); + let namespace_id = NamespaceId::from_external_id(id, uuid); + let create_namespace_op = + ChronicleOperation::CreateNamespace(CreateNamespace::new(namespace_id.clone())); + Ok((namespace_id, vec![create_namespace_op])) + }, } } @@ -974,10 +1038,11 @@ where /// /// We use our local store to see if the agent already exists, disambiguating the URI if so #[instrument(skip(self))] + #[instrument(skip(self))] async fn create_entity( &self, - external_id: ExternalId, - namespace: ExternalId, + id: EntityId, + namespace_id: ExternalId, attributes: Attributes, identity: AuthId, ) -> Result { @@ -986,21 +1051,19 @@ where let mut connection = api.store.connection()?; connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; + let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace_id)?; let applying_new_namespace = !to_apply.is_empty(); - let id = EntityId::from_external_id(&external_id); - let create = ChronicleOperation::EntityExists(EntityExists { namespace: namespace.clone(), - external_id: external_id.clone(), + id: id.clone(), }); to_apply.push(create); let set_type = ChronicleOperation::SetAttributes(SetAttributes::Entity { - id: EntityId::from_external_id(&external_id), + id: id.clone(), namespace, attributes, }); @@ -1025,8 +1088,8 @@ where #[instrument(skip(self))] async fn create_activity( &self, - external_id: ExternalId, - namespace: ExternalId, + activity_id: ExternalId, + namespace_id: ExternalId, attributes: Attributes, identity: AuthId, ) -> Result { @@ -1035,20 +1098,19 @@ where let mut connection = api.store.connection()?; connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; + let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace_id)?; let applying_new_namespace = !to_apply.is_empty(); let create = ChronicleOperation::ActivityExists(ActivityExists { namespace: namespace.clone(), - external_id: external_id.clone(), + id: ActivityId::from_external_id(&activity_id), }); to_apply.push(create); - let id = ActivityId::from_external_id(&external_id); let set_type = ChronicleOperation::SetAttributes(SetAttributes::Activity { - id: id.clone(), + id: ActivityId::from_external_id(&activity_id), namespace, attributes, }); @@ -1057,7 +1119,7 @@ where api.apply_effects_and_submit( connection, - id, + ActivityId::from_external_id(&activity_id), identity, to_apply, applying_new_namespace, @@ -1073,7 +1135,7 @@ where #[instrument(skip(self))] async fn create_agent( &self, - external_id: ExternalId, + agent_id: ExternalId, namespace: ExternalId, attributes: Attributes, identity: AuthId, @@ -1088,13 +1150,13 @@ where let applying_new_namespace = !to_apply.is_empty(); let create = ChronicleOperation::AgentExists(AgentExists { - external_id: external_id.to_owned(), + id: AgentId::from_external_id(&agent_id), namespace: namespace.clone(), }); to_apply.push(create); - let id = AgentId::from_external_id(&external_id); + let id = AgentId::from_external_id(&agent_id); let set_type = ChronicleOperation::SetAttributes(SetAttributes::Agent { id: id.clone(), namespace, @@ -1119,15 +1181,15 @@ where /// not already exist in local storage async fn create_namespace( &self, - external_id: &ExternalId, + name: &ExternalId, identity: AuthId, ) -> Result { let mut api = self.clone(); - let external_id = external_id.to_owned(); + let name = name.to_owned(); tokio::task::spawn_blocking(move || { let mut connection = api.store.connection()?; connection.build_transaction().run(|connection| { - let (namespace, to_apply) = api.ensure_namespace(connection, &external_id)?; + let (namespace, to_apply) = api.ensure_namespace(connection, &name)?; api.submit(namespace, identity, to_apply) }) @@ -1176,18 +1238,21 @@ where #[instrument(skip(self))] async fn dispatch(&mut self, command: (ApiCommand, AuthId)) -> Result { match command { - (ApiCommand::DepthCharge(DepthChargeCommand { namespace }), identity) => - self.depth_charge(namespace, identity).await, - (ApiCommand::Import(ImportCommand { namespace, operations }), identity) => - self.submit_import_operations(identity, namespace, operations).await, - (ApiCommand::NameSpace(NamespaceCommand::Create { external_id }), identity) => - self.create_namespace(&external_id, identity).await, - ( - ApiCommand::Agent(AgentCommand::Create { external_id, namespace, attributes }), - identity, - ) => self.create_agent(external_id, namespace, attributes, identity).await, - (ApiCommand::Agent(AgentCommand::UseInContext { id, namespace }), _identity) => - self.use_agent_in_cli_context(id, namespace).await, + (ApiCommand::DepthCharge(DepthChargeCommand { namespace }), identity) => { + self.depth_charge(namespace, identity).await + }, + (ApiCommand::Import(ImportCommand { operations }), identity) => { + self.submit_import_operations(identity, operations).await + }, + (ApiCommand::NameSpace(NamespaceCommand::Create { id }), identity) => { + self.create_namespace(&id, identity).await + }, + (ApiCommand::Agent(AgentCommand::Create { id, namespace, attributes }), identity) => { + self.create_agent(id, namespace, attributes, identity).await + }, + (ApiCommand::Agent(AgentCommand::UseInContext { id, namespace }), _identity) => { + self.use_agent_in_cli_context(id, namespace).await + }, ( ApiCommand::Agent(AgentCommand::Delegate { id, @@ -1199,13 +1264,9 @@ where identity, ) => self.delegate(namespace, id, delegate, activity, role, identity).await, ( - ApiCommand::Activity(ActivityCommand::Create { - external_id, - namespace, - attributes, - }), + ApiCommand::Activity(ActivityCommand::Create { id, namespace, attributes }), identity, - ) => self.create_activity(external_id, namespace, attributes, identity).await, + ) => self.create_activity(id, namespace, attributes, identity).await, ( ApiCommand::Activity(ActivityCommand::Instant { id, namespace, time, agent }), identity, @@ -1218,8 +1279,9 @@ where ApiCommand::Activity(ActivityCommand::End { id, namespace, time, agent }), identity, ) => self.end_activity(id, namespace, time, agent, identity).await, - (ApiCommand::Activity(ActivityCommand::Use { id, namespace, activity }), identity) => - self.activity_use(id, namespace, activity, identity).await, + (ApiCommand::Activity(ActivityCommand::Use { id, namespace, activity }), identity) => { + self.activity_use(id, namespace, activity, identity).await + }, ( ApiCommand::Activity(ActivityCommand::WasInformedBy { id, @@ -1241,10 +1303,10 @@ where ApiCommand::Entity(EntityCommand::Attribute { id, namespace, responsible, role }), identity, ) => self.attribute(namespace, responsible, id, role, identity).await, - ( - ApiCommand::Entity(EntityCommand::Create { external_id, namespace, attributes }), - identity, - ) => self.create_entity(external_id, namespace, attributes, identity).await, + (ApiCommand::Entity(EntityCommand::Create { id, namespace, attributes }), identity) => { + self.create_entity(EntityId::from_external_id(&id), namespace, attributes, identity) + .await + }, ( ApiCommand::Activity(ActivityCommand::Generate { id, namespace, activity }), identity, @@ -1258,9 +1320,10 @@ where derivation, }), identity, - ) => + ) => { self.entity_derive(id, namespace, activity, used_entity, derivation, identity) - .await, + .await + }, (ApiCommand::Query(query), _identity) => self.query(query).await, } } @@ -1285,13 +1348,13 @@ where let applying_new_namespace = !to_apply.is_empty(); - let tx = ChronicleOperation::AgentActsOnBehalfOf(ActsOnBehalfOf::new( - &namespace, - &responsible_id, - &delegate_id, - activity_id.as_ref(), + let tx = ChronicleOperation::agent_acts_on_behalf_of( + namespace, + responsible_id.clone(), + delegate_id, + activity_id, role, - )); + ); to_apply.push(tx); @@ -1326,12 +1389,12 @@ where let applying_new_namespace = !to_apply.is_empty(); - let tx = ChronicleOperation::WasAssociatedWith(WasAssociatedWith::new( - &namespace, - &activity_id, - &responsible_id, + let tx = ChronicleOperation::was_associated_with( + namespace, + activity_id, + responsible_id.clone(), role, - )); + ); to_apply.push(tx); @@ -1366,12 +1429,12 @@ where let applying_new_namespace = !to_apply.is_empty(); - let tx = ChronicleOperation::WasAttributedTo(WasAttributedTo::new( - &namespace, - &entity_id, - &responsible_id, + let tx = ChronicleOperation::was_attributed_to( + namespace, + entity_id, + responsible_id.clone(), role, - )); + ); to_apply.push(tx); @@ -1445,7 +1508,6 @@ where async fn submit_import_operations( &self, identity: AuthId, - namespace: NamespaceId, operations: Vec, ) -> Result { let mut api = self.clone(); @@ -1464,8 +1526,7 @@ where Ok(ApiResponse::import_submitted(model, tx_id)) } else { info!("Import will not result in any data changes"); - let model = ProvModel::from_tx(&operations)?; - Ok(ApiResponse::already_recorded(namespace, model)) + Ok(ApiResponse::AlreadyRecordedAll) } }) }) @@ -1520,22 +1581,27 @@ where } }; + let now = Utc::now(); + to_apply.push(ChronicleOperation::StartActivity(StartActivity { namespace: namespace.clone(), id: id.clone(), - time: time.unwrap_or_else(Utc::now).into(), + time: time.unwrap_or(now).into(), })); to_apply.push(ChronicleOperation::EndActivity(EndActivity { namespace: namespace.clone(), id: id.clone(), - time: time.unwrap_or_else(Utc::now).into(), + time: time.unwrap_or(now).into(), })); if let Some(agent_id) = agent_id { - to_apply.push(ChronicleOperation::WasAssociatedWith(WasAssociatedWith::new( - &namespace, &id, &agent_id, None, - ))); + to_apply.push(ChronicleOperation::was_associated_with( + namespace, + id.clone(), + agent_id, + None, + )); } api.apply_effects_and_submit( @@ -1587,9 +1653,12 @@ where })); if let Some(agent_id) = agent_id { - to_apply.push(ChronicleOperation::WasAssociatedWith(WasAssociatedWith::new( - &namespace, &id, &agent_id, None, - ))); + to_apply.push(ChronicleOperation::was_associated_with( + namespace, + id.clone(), + agent_id, + None, + )); } api.apply_effects_and_submit( @@ -1641,9 +1710,12 @@ where })); if let Some(agent_id) = agent_id { - to_apply.push(ChronicleOperation::WasAssociatedWith(WasAssociatedWith::new( - &namespace, &id, &agent_id, None, - ))); + to_apply.push(ChronicleOperation::was_associated_with( + namespace, + id.clone(), + agent_id, + None, + )); } api.apply_effects_and_submit( diff --git a/crates/chronicle-arrow/Cargo.toml b/crates/chronicle-arrow/Cargo.toml new file mode 100644 index 000000000..4cbe91e5d --- /dev/null +++ b/crates/chronicle-arrow/Cargo.toml @@ -0,0 +1,44 @@ +[package] +edition = "2021" +name = "chronicle-arrow" +version = "0.1.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +arrow-array = { workspace = true } +arrow-flight = { workspace = true } +arrow-ipc = { workspace = true } +arrow-schema = { workspace = true } +futures = { workspace = true } +lazy_static = { workspace = true } +prost-types = { version = "0.12.3", default-features = false } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { version = "^1", default-features = false, features = [ + "macros", + "rt", + "rt-multi-thread", +] } +tonic = { version = "^0.10.0", default-features = false, features = [ + "transport", + "codegen", + "prost", +] } +tracing = { workspace = true } +uuid = { workspace = true } + + +#local dependencies +api = { path = "../api" } +chronicle-persistence = { path = "../chronicle-persistence" } +common = { path = "../common", features = ["std"] } + + +[dev-dependencies] +chronicle-telemetry = { path = "../chronicle-telemetry" } +chronicle-test-infrastructure = { path = "../chronicle-test-infrastructure" } +insta = { workspace = true, features = ["toml"] } +portpicker = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_arrow = { version = "*", features = ["arrow-50"] } diff --git a/crates/chronicle-arrow/src/lib.rs b/crates/chronicle-arrow/src/lib.rs new file mode 100644 index 000000000..4f93a3132 --- /dev/null +++ b/crates/chronicle-arrow/src/lib.rs @@ -0,0 +1,865 @@ +use api::commands::{ApiCommand, ImportCommand}; +use api::{ApiDispatch, ApiError}; +use arrow_array::cast::AsArray; +use arrow_array::{ArrayRef, BooleanArray, Int64Array, NullArray, RecordBatch, StringArray}; +use arrow_flight::decode::FlightRecordBatchStream; + +use arrow_flight::{ + flight_service_server::FlightService, Action, ActionType, Criteria, Empty, FlightData, + FlightDescriptor, FlightInfo, HandshakeRequest, HandshakeResponse, PutResult, SchemaResult, + Ticket, +}; + +use arrow_flight::{IpcMessage, SchemaAsIpc}; +use arrow_schema::{ArrowError, Schema, SchemaBuilder}; + +use common::attributes::{Attribute, Attributes}; +use common::domain::{ + ActivityDef, AgentDef, AttributesTypeName, ChronicleDomainDef, EntityDef, PrimitiveType, + TypeName, +}; +use common::identity::AuthId; +use common::prov::operations::{ChronicleOperation, SetAttributes}; +use common::prov::{ActivityId, AgentId, DomaintypeId, EntityId, NamespaceId, ParseIriError}; +use futures::{stream, TryStreamExt}; +use futures::{stream::BoxStream, StreamExt}; +use lazy_static::lazy_static; +use std::collections::HashMap; +use std::net::SocketAddr; + +use std::sync::Arc; +use std::sync::Mutex; +use std::vec::Vec; +use tonic::transport::Server; +use tonic::{Request, Response, Status, Streaming}; +use tracing::{info, instrument}; + +use thiserror::Error; + +use crate::peekableflightdatastream::PeekableFlightDataStream; +mod peekableflightdatastream; + +#[derive(Error, Debug)] +pub enum ChronicleArrowError { + #[error("Arrow error: {0}")] + ArrowSchemaError( + #[from] + #[source] + ArrowError, + ), + #[error("Missing schema for the requested entity or activity")] + MissingSchemaError, + + #[error("Missing column: {0}")] + MissingColumn(String), + + #[error("Invalid descriptor path")] + InvalidDescriptorPath, + + #[error("Metadata not found")] + MetadataNotFound, + + #[error("API error: {0}")] + ApiError( + #[from] + #[source] + ApiError, + ), + #[error("Parse IRI : {0}")] + IriError( + #[from] + #[source] + ParseIriError, + ), +} + +#[derive(Clone)] +pub struct FlightServiceImpl { + domain: common::domain::ChronicleDomainDef, + store: chronicle_persistence::Store, + api: ApiDispatch, +} + +fn field_for_domain_primitive(prim: &PrimitiveType) -> Option { + match prim { + PrimitiveType::String => Some(arrow_schema::DataType::Utf8), + PrimitiveType::Int => Some(arrow_schema::DataType::Int64), + PrimitiveType::Bool => Some(arrow_schema::DataType::Boolean), + _ => None, + } +} + +fn schema_for_entity(entity: &EntityDef) -> Schema { + let mut builder = SchemaBuilder::new(); + + builder.push(arrow_schema::Field::new("namespace_id", arrow_schema::DataType::Utf8, false)); + + builder.push(arrow_schema::Field::new("id", arrow_schema::DataType::Utf8, false)); + for attribute in &entity.attributes { + if let Some(data_type) = field_for_domain_primitive(&attribute.primitive_type) { + builder.push(arrow_schema::Field::new( + &attribute.preserve_inflection(), + data_type, + true, + )); + } + } + builder.finish() +} + +fn schema_for_activity(activity: &ActivityDef) -> Schema { + let mut builder = SchemaBuilder::new(); + + builder.push(arrow_schema::Field::new("namespace_id", arrow_schema::DataType::Utf8, false)); + builder.push(arrow_schema::Field::new("id", arrow_schema::DataType::Utf8, false)); + builder.push(arrow_schema::Field::new( + "started", + arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Millisecond, None), + true, + )); + builder.push(arrow_schema::Field::new( + "ended", + arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Millisecond, None), + true, + )); + + for attribute in &activity.attributes { + if let Some(typ) = field_for_domain_primitive(&attribute.primitive_type) { + builder.push(arrow_schema::Field::new(&attribute.preserve_inflection(), typ, true)); + } + } + + builder.push(arrow_schema::Field::new( + "was_associated_with", + arrow_schema::DataType::List(Arc::new(arrow_schema::Field::new( + "agent", + arrow_schema::DataType::Utf8, + false, + ))), + false, + )); + builder.push(arrow_schema::Field::new( + "used", + arrow_schema::DataType::List(Arc::new(arrow_schema::Field::new( + "entity", + arrow_schema::DataType::Utf8, + false, + ))), + true, + )); + builder.push(arrow_schema::Field::new( + "was_informed_by", + arrow_schema::DataType::List(Arc::new(arrow_schema::Field::new( + "activity", + arrow_schema::DataType::Utf8, + false, + ))), + false, + )); + builder.push(arrow_schema::Field::new( + "generated", + arrow_schema::DataType::List(Arc::new(arrow_schema::Field::new( + "entity", + arrow_schema::DataType::Utf8, + true, + ))), + true, + )); + builder.finish() +} + +fn schema_for_agent(agent: &AgentDef) -> Schema { + let mut builder = SchemaBuilder::new(); + builder.push(arrow_schema::Field::new("namespace_id", arrow_schema::DataType::Utf8, false)); + builder.push(arrow_schema::Field::new("id", arrow_schema::DataType::Utf8, false)); + for attribute in &agent.attributes { + if let Some(typ) = field_for_domain_primitive(&attribute.primitive_type) { + builder.push(arrow_schema::Field::new(&attribute.preserve_inflection(), typ, true)); + } + } + + builder.finish() +} + +#[derive(PartialEq, Eq)] +enum Term { + Entity, + Activity, + Agent, +} + +struct DomainTypeMeta { + pub schema: Schema, + pub term: Term, + pub typ: Option, +} + +lazy_static! { + static ref SCHEMA_CACHE: Mutex, Arc>> = + Mutex::new(HashMap::new()); +} + +fn get_domain_type_meta_from_cache(descriptor_path: &Vec) -> Option> { + let cache = SCHEMA_CACHE.lock().unwrap(); + cache.get(descriptor_path).cloned() +} + +fn cache_metadata( + term: Term, + domain_type_id: DomaintypeId, + descriptor_path: Vec, + schema: Schema, +) { + let mut cache = SCHEMA_CACHE.lock().unwrap(); + let domain_type_meta = Arc::new(DomainTypeMeta { schema, term, typ: Some(domain_type_id) }); + cache.insert(descriptor_path, domain_type_meta); +} + +#[tracing::instrument(skip(domain_def))] +fn cache_domain_schemas(domain_def: &ChronicleDomainDef) { + for entity in &domain_def.entities { + let schema = schema_for_entity(entity); + let descriptor_path = vec![entity.as_type_name()]; + cache_metadata( + Term::Entity, + DomaintypeId::from_external_id(entity.as_type_name()), + descriptor_path, + schema, + ); + } + + for agent in &domain_def.agents { + let schema = schema_for_agent(agent); + let descriptor_path = vec![agent.as_type_name()]; + cache_metadata( + Term::Agent, + DomaintypeId::from_external_id(agent.as_type_name()), + descriptor_path, + schema, + ); + } + + for activity in &domain_def.activities { + let schema = schema_for_activity(activity); + let descriptor_path = vec![activity.as_type_name()]; + cache_metadata( + Term::Activity, + DomaintypeId::from_external_id(activity.as_type_name()), + descriptor_path, + schema, + ); + } +} + +#[tracing::instrument(skip(record_batch))] +async fn process_record_batch( + descriptor_path: &Vec, + record_batch: RecordBatch, + api: &ApiDispatch, +) -> Result<(), ChronicleArrowError> { + if descriptor_path.is_empty() { + return Err(ChronicleArrowError::InvalidDescriptorPath); + } + + let domain_type_meta = get_domain_type_meta_from_cache(descriptor_path) + .ok_or(ChronicleArrowError::MetadataNotFound)?; + + let attribute_columns = domain_type_meta + .schema + .fields() + .iter() + .filter_map(|field| { + if field.name().ends_with("Attribute") { + Some(field.name().clone()) + } else { + None + } + }) + .collect::>(); + + tracing::debug!(?attribute_columns, "Extracted attribute column names"); + match domain_type_meta.term { + Term::Entity => { + create_chronicle_entity(&domain_type_meta.typ, &record_batch, &attribute_columns, api) + .await? + }, + Term::Activity => { + create_chronicle_activity(&domain_type_meta.typ, &record_batch, &attribute_columns, api) + .await? + }, + Term::Agent => { + create_chronicle_agent(&domain_type_meta.typ, &record_batch, &attribute_columns, api) + .await? + }, + } + Ok(()) +} + +async fn create_chronicle_entity( + domain_type: &Option, + record_batch: &RecordBatch, + attribute_columns: &Vec, + api: &ApiDispatch, +) -> Result<(), ChronicleArrowError> { + create_chronicle_terms(record_batch, Term::Entity, domain_type, attribute_columns, api).await +} + +async fn create_chronicle_activity( + domain_type: &Option, + record_batch: &RecordBatch, + attribute_columns: &Vec, + api: &ApiDispatch, +) -> Result<(), ChronicleArrowError> { + create_chronicle_terms(record_batch, Term::Activity, domain_type, attribute_columns, api).await +} + +async fn create_chronicle_agent( + domain_type: &Option, + record_batch: &RecordBatch, + attribute_columns: &Vec, + api: &ApiDispatch, +) -> Result<(), ChronicleArrowError> { + create_chronicle_terms(record_batch, Term::Agent, domain_type, attribute_columns, api).await +} + +async fn create_chronicle_terms( + record_batch: &RecordBatch, + record_type: Term, + domain_type: &Option, + attribute_columns: &Vec, + api: &ApiDispatch, +) -> Result<(), ChronicleArrowError> { + let ns_column = record_batch + .column_by_name("namespace_id") + .ok_or(ChronicleArrowError::MissingColumn("namespace_id".to_string()))?; + + let id_column = record_batch + .column_by_name("id") + .ok_or(ChronicleArrowError::MissingColumn("id".to_string()))?; + + let attribute_columns_refs: Vec<&String> = attribute_columns.iter().collect(); + let attribute_values = attribute_columns_refs + .iter() + .map(|column_name| (column_name.to_string(), record_batch.column_by_name(column_name))) + .filter_map(|(column_name, array_ref)| array_ref.map(|array_ref| (column_name, array_ref))) + .collect::>(); + + tracing::debug!(?attribute_columns, "Processing attribute columns"); + + let mut operations = Vec::new(); + + for row_index in 0..record_batch.num_rows() { + let ns: NamespaceId = NamespaceId::try_from(ns_column.as_string::().value(row_index))?; + let id = id_column.as_string::().value(row_index); + + let mut attributes: Vec<(String, Attribute)> = vec![]; + + for (attribute_name, attribute_array) in attribute_values.iter() { + tracing::trace!(%attribute_name, "Appending to attributes"); + if let Some(array) = attribute_array.as_any().downcast_ref::() { + let value = array.value(row_index); + attributes.push(( + attribute_name.clone(), + Attribute::new( + attribute_name.clone(), + serde_json::Value::String(value.to_string()), + ), + )); + } else if let Some(array) = attribute_array.as_any().downcast_ref::() { + let value = array.value(row_index); + attributes.push(( + attribute_name.clone(), + Attribute::new(attribute_name.clone(), serde_json::Value::Number(value.into())), + )); + } else if let Some(array) = attribute_array.as_any().downcast_ref::() { + let value = array.value(row_index); + attributes.push(( + attribute_name.clone(), + Attribute::new(attribute_name.clone(), serde_json::Value::Bool(value)), + )); + } else { + tracing::warn!(%attribute_name, "Unsupported attribute type"); + } + } + + let attributes = Attributes::new(domain_type.clone(), attributes); + + match record_type { + Term::Entity => { + operations.push(ChronicleOperation::entity_exists( + ns.clone(), + EntityId::from_external_id(id), + )); + operations.push(ChronicleOperation::set_attributes(SetAttributes::entity( + ns.clone(), + EntityId::from_external_id(id), + attributes, + ))); + }, + Term::Activity => { + operations.push(ChronicleOperation::activity_exists( + ns.clone(), + ActivityId::from_external_id(id), + )); + operations.push(ChronicleOperation::set_attributes(SetAttributes::activity( + ns.clone(), + ActivityId::from_external_id(id), + attributes, + ))); + }, + Term::Agent => { + operations.push(ChronicleOperation::agent_exists( + ns.clone(), + AgentId::from_external_id(id), + )); + operations.push(ChronicleOperation::set_attributes(SetAttributes::agent( + ns.clone(), + AgentId::from_external_id(id), + attributes, + ))); + }, + } + } + + tracing::debug!("Generated operations for {:?}", &operations); + + api.dispatch(ApiCommand::Import(ImportCommand { operations }), AuthId::anonymous()) + .await?; + + Ok(()) +} + +#[tonic::async_trait] +impl FlightService for FlightServiceImpl { + type DoActionStream = BoxStream<'static, Result>; + type DoExchangeStream = BoxStream<'static, Result>; + type DoGetStream = BoxStream<'static, Result>; + type DoPutStream = BoxStream<'static, Result>; + type HandshakeStream = BoxStream<'static, Result>; + type ListActionsStream = BoxStream<'static, Result>; + type ListFlightsStream = BoxStream<'static, Result>; + + async fn handshake( + &self, + _request: Request>, + ) -> Result, Status> { + Ok(Response::new(Box::pin(futures::stream::empty()) as Self::HandshakeStream)) + } + + async fn list_flights( + &self, + _request: Request, + ) -> Result, Status> { + let entity_flights: Vec<_> = self + .domain + .entities + .iter() + .map(|entity| { + let descriptor_path = vec![entity.as_type_name()]; + let schema = get_domain_type_meta_from_cache(&descriptor_path) + .ok_or_else(|| ChronicleArrowError::MissingSchemaError)?; + FlightInfo::new() + .with_descriptor(FlightDescriptor::new_path(descriptor_path)) + .try_with_schema(&schema.schema) + .map_err(ChronicleArrowError::from) + }) + .collect::, _>>() + .map_err(|e| Status::from_error(Box::new(e)))?; + + let agent_flights: Vec<_> = self + .domain + .agents + .iter() + .map(|agent| { + let descriptor_path = vec![agent.as_type_name()]; + let schema = get_domain_type_meta_from_cache(&descriptor_path) + .ok_or_else(|| ChronicleArrowError::MissingSchemaError)?; + FlightInfo::new() + .with_descriptor(FlightDescriptor::new_path(descriptor_path)) + .try_with_schema(&schema.schema) + .map_err(ChronicleArrowError::from) + }) + .collect::, _>>() + .map_err(|e| Status::from_error(Box::new(e)))?; + + let activity_flights: Vec<_> = self + .domain + .activities + .iter() + .map(|activity| { + let descriptor_path = vec![activity.as_type_name()]; + let schema = get_domain_type_meta_from_cache(&descriptor_path) + .ok_or_else(|| ChronicleArrowError::MissingSchemaError)?; + FlightInfo::new() + .with_descriptor(FlightDescriptor::new_path(descriptor_path)) + .try_with_schema(&schema.schema) + .map_err(ChronicleArrowError::from) + }) + .collect::, _>>() + .map_err(|e| Status::from_error(Box::new(e)))?; + + let combined_flights = entity_flights + .into_iter() + .chain(agent_flights) + .chain(activity_flights) + .collect::>(); + + let stream = futures::stream::iter(combined_flights.into_iter().map(Ok)); + Ok(Response::new(Box::pin(stream) as Self::ListFlightsStream)) + } + + #[instrument(skip(self, _request))] + async fn get_flight_info( + &self, + _request: Request, + ) -> Result, Status> { + let descriptor = _request.into_inner(); + let path = descriptor.path; + if path.is_empty() { + return Err(Status::invalid_argument("Descriptor path is empty")); + } + + let type_name = &path[0]; + let descriptor_path = vec![type_name.to_string()]; + let schema = get_domain_type_meta_from_cache(&descriptor_path) + .ok_or_else(|| ChronicleArrowError::MissingSchemaError) + .map_err(|e| Status::internal(format!("Failed to get cached schema: {}", e)))?; + + let flight_info = FlightInfo::new() + .with_descriptor(FlightDescriptor::new_path(descriptor_path)) + .try_with_schema(&schema.schema) + .map_err(|e| Status::from_error(e.into()))?; + + Ok(Response::new(flight_info)) + } + + #[instrument(skip(self, _request))] + async fn get_schema( + &self, + _request: Request, + ) -> Result, Status> { + let descriptor = _request.into_inner(); + let path = descriptor.path; + if path.is_empty() { + return Err(Status::invalid_argument("Descriptor path is empty")); + } + + let type_name = &path[0]; + let descriptor_path = vec![type_name.to_string()]; + let schema = get_domain_type_meta_from_cache(&descriptor_path) + .ok_or_else(|| ChronicleArrowError::MissingSchemaError) + .map_err(|e| Status::internal(format!("Failed to get cached schema: {}", e)))?; + + let options = arrow_ipc::writer::IpcWriteOptions::default(); + let ipc_message_result = SchemaAsIpc::new(&schema.schema, &options).try_into(); + match ipc_message_result { + Ok(IpcMessage(schema)) => Ok(Response::new(SchemaResult { schema })), + Err(e) => { + Err(Status::internal(format!("Failed to convert schema to IPC message: {}", e))) + }, + } + } + async fn do_get( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Implement do_get")) + } + + #[instrument(skip(self, request))] + async fn do_put( + &self, + request: Request>, + ) -> Result, Status> { + let mut stream = request.map(PeekableFlightDataStream::new).into_inner(); + let first_item = stream.peek().await; + + let flight_descriptor = match &first_item { + Some(Ok(flight_data)) => match flight_data.flight_descriptor.clone() { + Some(descriptor) => descriptor, + None => return Err(Status::invalid_argument("Flight data has no descriptor")), + }, + Some(Err(e)) => { + return Err(Status::internal(format!( + "Failed to get first item from stream: {}", + e + ))) + }, + None => { + return Err(Status::invalid_argument("Stream is empty")); + }, + }; + + tracing::debug!( + descriptor_type = %flight_descriptor.r#type, + descriptor_cmd = %String::from_utf8_lossy(&flight_descriptor.cmd), + descriptor_path = ?flight_descriptor.path, + "Flight descriptor properties" + ); + + let filtered_stream = stream.filter_map(|item| async move { + match item { + Ok(flight_data) => { + tracing::trace!("Processing flight data item {:?}", flight_data); + Some(Ok(flight_data)) + }, + Err(e) => { + tracing::error!(error = %e, "Error processing stream item."); + None + }, + } + }); + + let mut decoder = FlightRecordBatchStream::new_from_flight_data(filtered_stream); + while let Some(batch) = decoder.next().await { + tracing::debug!("Processing batch: {:?}", batch); + let batch = batch?; + + process_record_batch(&flight_descriptor.path, batch, &self.api) + .await + .map_err(|e| Status::from_error(e.into()))?; + } + Ok(Response::new(Box::pin(stream::empty()) as Self::DoPutStream)) + } + + #[tracing::instrument(skip(self, _request))] + async fn do_action( + &self, + _request: Request, + ) -> Result, Status> { + tracing::info!("No actions available, returning empty stream."); + Ok(Response::new(Box::pin(stream::empty()))) + } + + #[tracing::instrument(skip(self, _request))] + async fn list_actions( + &self, + _request: Request, + ) -> Result, Status> { + tracing::info!("No actions available."); + Ok(Response::new(Box::pin(stream::empty()))) + } + + async fn do_exchange( + &self, + _request: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("Implement do_exchange")) + } +} + +#[instrument(skip(store, domain, api))] +pub async fn run_flight_service( + store: chronicle_persistence::Store, + domain: common::domain::ChronicleDomainDef, + api: ApiDispatch, + addr: SocketAddr, +) -> Result<(), Box> { + cache_domain_schemas(&domain); + let flight_service = FlightServiceImpl { store, domain, api }; + + info!("Starting flight service at {}", addr); + + Server::builder() + .add_service(arrow_flight::flight_service_server::FlightServiceServer::new(flight_service)) + .serve(addr) + .await?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + + use arrow_array::RecordBatch; + use arrow_flight::{ + flight_service_client::FlightServiceClient, Criteria, FlightData, FlightDescriptor, + SchemaAsIpc, + }; + + use arrow_ipc::writer::{self, IpcWriteOptions}; + use arrow_schema::ArrowError; + use chronicle_persistence::Store; + use chronicle_test_infrastructure::substitutes::{test_api, TestDispatch}; + use common::{domain::ChronicleDomainDef, prov::NamespaceId}; + use futures::{stream, StreamExt}; + use portpicker::pick_unused_port; + use serde::{Deserialize, Serialize}; + + use std::{net::SocketAddr, sync::Arc, time::Duration}; + use tonic::Request; + use uuid::Uuid; + + use crate::{cache_domain_schemas, get_domain_type_meta_from_cache, DomainTypeMeta}; + + async fn setup_test_environment<'a>( + domain: &ChronicleDomainDef, + ) -> Result< + (FlightServiceClient, TestDispatch<'a>), + Box, + > { + chronicle_telemetry::telemetry(None, chronicle_telemetry::ConsoleLogging::Pretty); + let api = test_api().await; + let port = pick_unused_port().expect("No ports free"); + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + let store = Store::new(api.temporary_database().connection_pool().unwrap()).unwrap(); + let dispatch = api.api_dispatch().clone(); + let domain = domain.clone(); + tokio::spawn(async move { + super::run_flight_service(store, domain, dispatch, addr).await.unwrap(); + }); + + tokio::time::sleep(Duration::from_secs(5)).await; + + let client = FlightServiceClient::connect(format!("http://{}", addr)).await?; + Ok((client, api)) + } + + #[tracing::instrument] + fn create_test_domain_def() -> ChronicleDomainDef { + let yaml = r#" +name: Manufacturing +attributes: + BatchID: + type: String + CertID: + type: String + CompanyName: + type: String + PartID: + type: String + Location: + type: String +agents: + Contractor: + attributes: + - CompanyName + - Location +entities: + Certificate: + attributes: + - CertID + Item: + attributes: + - PartID +activities: + ItemCertified: + attributes: [] + ItemManufactured: + attributes: + - BatchID +roles: + - CERTIFIER + - MANUFACTURER +"#; + + ChronicleDomainDef::from_input_string(yaml).unwrap() + } + + #[derive(Serialize, Deserialize)] + struct Certificate { + namespace_id: String, + id: String, + certIDAttribute: String, + } + + #[derive(Serialize, Deserialize)] + struct Item { + namespace_id: String, + id: String, + partIDAttribute: String, + } + + fn create_test_entity_certificate(meta: &DomainTypeMeta, count: u32) -> RecordBatch { + let fields = meta.schema.all_fields().into_iter().cloned().collect::>(); + + let mut certs = Vec::new(); + for i in 0..count { + certs.push(Certificate { + namespace_id: NamespaceId::from_external_id("default", Uuid::default()).to_string(), + id: format!("certificate-{}", i), + certIDAttribute: format!("CERT-{}", i), + }); + } + + let arrays = serde_arrow::to_arrow(&fields, &certs).unwrap(); + + RecordBatch::try_new(Arc::new(meta.schema.clone()), arrays).unwrap() + } + + fn create_test_entity_item(meta: &DomainTypeMeta, count: u32) -> RecordBatch { + let fields = meta.schema.all_fields().into_iter().cloned().collect::>(); + + let mut certs = Vec::new(); + for i in 0..count { + certs.push(Item { + namespace_id: NamespaceId::from_external_id("default", Uuid::default()).to_string(), + id: format!("item-{}", i), + partIDAttribute: format!("PART-{}", i), + }); + } + + let arrays = serde_arrow::to_arrow(&fields, &certs).unwrap(); + + RecordBatch::try_new(Arc::new(meta.schema.clone()), arrays).unwrap() + } + + pub fn batches_to_flight_data( + descriptor: &FlightDescriptor, + meta: &DomainTypeMeta, + batches: Vec, + ) -> Result, ArrowError> { + let options = IpcWriteOptions::default(); + let schema_flight_data: FlightData = + std::convert::Into::::into(SchemaAsIpc::new(&meta.schema, &options)) + .with_descriptor(descriptor.clone()); + let mut dictionaries = vec![]; + let mut flight_data = vec![]; + + let data_gen = writer::IpcDataGenerator::default(); + let mut dictionary_tracker = writer::DictionaryTracker::new(false); + + for batch in batches.iter() { + let (encoded_dictionaries, encoded_batch) = + data_gen.encoded_batch(batch, &mut dictionary_tracker, &options)?; + + dictionaries.extend(encoded_dictionaries.into_iter().map(Into::into)); + let next: FlightData = encoded_batch.into(); + flight_data.push(next); + } + let mut stream = vec![schema_flight_data]; + stream.extend(dictionaries); + stream.extend(flight_data); + let flight_data: Vec<_> = stream.into_iter().collect(); + Ok(flight_data) + } + + async fn create_test_flight_data() -> Result, Box> { + let path = vec!["CertificateEntity".to_owned()]; + let meta = get_domain_type_meta_from_cache(&path).unwrap(); + let batch = create_test_entity_certificate( + &get_domain_type_meta_from_cache(&path).unwrap().clone(), + 10, + ); + + let flight_data = + batches_to_flight_data(&FlightDescriptor::new_path(path.clone()), &meta, vec![batch]) + .unwrap(); + + Ok(flight_data) + } + + #[tokio::test] + async fn flight_service_is_isomorphic() -> Result<(), Box> { + cache_domain_schemas(&create_test_domain_def()); + let domain = create_test_domain_def(); + let (mut client, _api) = setup_test_environment(&domain).await?; + + client + .do_put(stream::iter(create_test_flight_data().await.unwrap()).boxed()) + .await + .unwrap(); + + Ok(()) + } +} diff --git a/crates/chronicle-arrow/src/peekableflightdatastream.rs b/crates/chronicle-arrow/src/peekableflightdatastream.rs new file mode 100644 index 000000000..3045df5f3 --- /dev/null +++ b/crates/chronicle-arrow/src/peekableflightdatastream.rs @@ -0,0 +1,89 @@ +use std::pin::Pin; + +use arrow_flight::FlightData; +use futures::{stream::Peekable, Stream, StreamExt}; +use tonic::{Status, Streaming}; + +/// A wrapper around [`Streaming`] that allows "peeking" at the +/// message at the front of the stream without consuming it. +/// This is needed because sometimes the first message in the stream will contain +/// a [`FlightDescriptor`] in addition to potentially any data, and the dispatch logic +/// must inspect this information. +/// +/// # Example +/// +/// [`PeekableFlightDataStream::peek`] can be used to peek at the first message without +/// discarding it; otherwise, `PeekableFlightDataStream` can be used as a regular stream. +/// See the following example: +/// +/// ```no_run +/// use arrow_array::RecordBatch; +/// use arrow_flight::decode::FlightRecordBatchStream; +/// use arrow_flight::FlightDescriptor; +/// use arrow_flight::error::FlightError; +/// use PeekableFlightDataStream; +/// use tonic::{Request, Status}; +/// use futures::TryStreamExt; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Status> { +/// let request: Request = todo!(); +/// let stream: PeekableFlightDataStream = request.into_inner(); +/// +/// // The first message contains the flight descriptor and the schema. +/// // Read the flight descriptor without discarding the schema: +/// let flight_descriptor: FlightDescriptor = stream +/// .peek() +/// .await +/// .cloned() +/// .transpose()? +/// .and_then(|data| data.flight_descriptor) +/// .expect("first message should contain flight descriptor"); +/// +/// // Pass the stream through a decoder +/// let batches: Vec = FlightRecordBatchStream::new_from_flight_data( +/// request.into_inner().map_err(|e| e.into()), +/// ) +/// .try_collect() +/// .await?; +/// } +/// ``` +pub struct PeekableFlightDataStream { + inner: Peekable>, +} + +impl PeekableFlightDataStream { + pub fn new(stream: Streaming) -> Self { + Self { inner: stream.peekable() } + } + + /// Convert this stream into a `Streaming`. + /// Any messages observed through [`Self::peek`] will be lost + /// after the conversion. + pub fn into_inner(self) -> Streaming { + self.inner.into_inner() + } + + /// Convert this stream into a `Peekable>`. + /// Preserves the state of the stream, so that calls to [`Self::peek`] + /// and [`Self::poll_next`] are the same. + pub fn into_peekable(self) -> Peekable> { + self.inner + } + + /// Peek at the head of this stream without advancing it. + pub async fn peek(&mut self) -> Option<&Result> { + Pin::new(&mut self.inner).peek().await + } +} + +impl Stream for PeekableFlightDataStream { + type Item = Result; + + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.inner.poll_next_unpin(cx) + } +} diff --git a/crates/chronicle-persistence/Cargo.toml b/crates/chronicle-persistence/Cargo.toml new file mode 100644 index 000000000..72fc6e7e3 --- /dev/null +++ b/crates/chronicle-persistence/Cargo.toml @@ -0,0 +1,29 @@ +[package] +edition = "2021" +name = "chronicle-persistence" +version = "0.1.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-graphql = { workspace = true, features = [ + "chrono", + "unblock", + "default", + "uuid", +] } +async-trait = { workspace = true } +chrono = { workspace = true } +derivative = { workspace = true } +diesel = { workspace = true } +diesel_migrations = { workspace = true } +hex = { workspace = true } +r2d2 = { version = "^0.8.1" } +serde_json = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } + +#local dependencies +common = { path = "../common", features = ["diesel-bindings", "std"] } +protocol-substrate-chronicle = { path = "../protocol-substrate-chronicle" } diff --git a/crates/api/migrations/2022-01-18-144301_ledger_sync/down.sql b/crates/chronicle-persistence/migrations/2022-01-18-144301_ledger_sync/down.sql similarity index 100% rename from crates/api/migrations/2022-01-18-144301_ledger_sync/down.sql rename to crates/chronicle-persistence/migrations/2022-01-18-144301_ledger_sync/down.sql diff --git a/crates/api/migrations/2022-01-18-144301_ledger_sync/up.sql b/crates/chronicle-persistence/migrations/2022-01-18-144301_ledger_sync/up.sql similarity index 100% rename from crates/api/migrations/2022-01-18-144301_ledger_sync/up.sql rename to crates/chronicle-persistence/migrations/2022-01-18-144301_ledger_sync/up.sql diff --git a/crates/api/migrations/2023-04-11-111125_generation/down.sql b/crates/chronicle-persistence/migrations/2023-04-11-111125_generation/down.sql similarity index 100% rename from crates/api/migrations/2023-04-11-111125_generation/down.sql rename to crates/chronicle-persistence/migrations/2023-04-11-111125_generation/down.sql diff --git a/crates/api/migrations/2023-04-11-111125_generation/up.sql b/crates/chronicle-persistence/migrations/2023-04-11-111125_generation/up.sql similarity index 100% rename from crates/api/migrations/2023-04-11-111125_generation/up.sql rename to crates/chronicle-persistence/migrations/2023-04-11-111125_generation/up.sql diff --git a/crates/api/migrations/2023-06-16-145625_attribution_fix/down.sql b/crates/chronicle-persistence/migrations/2023-06-16-145625_attribution_fix/down.sql similarity index 100% rename from crates/api/migrations/2023-06-16-145625_attribution_fix/down.sql rename to crates/chronicle-persistence/migrations/2023-06-16-145625_attribution_fix/down.sql diff --git a/crates/api/migrations/2023-06-16-145625_attribution_fix/up.sql b/crates/chronicle-persistence/migrations/2023-06-16-145625_attribution_fix/up.sql similarity index 100% rename from crates/api/migrations/2023-06-16-145625_attribution_fix/up.sql rename to crates/chronicle-persistence/migrations/2023-06-16-145625_attribution_fix/up.sql diff --git a/crates/api/migrations/2023-07-12-123150_attachment/down.sql b/crates/chronicle-persistence/migrations/2023-07-12-123150_attachment/down.sql similarity index 100% rename from crates/api/migrations/2023-07-12-123150_attachment/down.sql rename to crates/chronicle-persistence/migrations/2023-07-12-123150_attachment/down.sql diff --git a/crates/api/migrations/2023-07-12-123150_attachment/up.sql b/crates/chronicle-persistence/migrations/2023-07-12-123150_attachment/up.sql similarity index 100% rename from crates/api/migrations/2023-07-12-123150_attachment/up.sql rename to crates/chronicle-persistence/migrations/2023-07-12-123150_attachment/up.sql diff --git a/crates/api/src/chronicle_graphql/cursor_query.rs b/crates/chronicle-persistence/src/cursor.rs similarity index 62% rename from crates/api/src/chronicle_graphql/cursor_query.rs rename to crates/chronicle-persistence/src/cursor.rs index 521c2fa8d..9a90f746b 100644 --- a/crates/api/src/chronicle_graphql/cursor_query.rs +++ b/crates/chronicle-persistence/src/cursor.rs @@ -1,9 +1,10 @@ -use async_graphql::{ - connection::{Edge, EmptyFields}, - OutputType, +use diesel::{ + pg::Pg, + prelude::*, + query_builder::*, + r2d2::{ConnectionManager, PooledConnection}, + sql_types::BigInt, }; -use diesel::{pg::Pg, prelude::*, query_builder::*, r2d2::ConnectionManager, sql_types::BigInt}; -use r2d2::PooledConnection; type Conn = PooledConnection>; @@ -12,8 +13,8 @@ const DEFAULT_PAGE_SIZE: i32 = 10; #[derive(QueryId)] pub struct CursorPosition { query: T, - pub(crate) start: i64, - pub(crate) limit: i64, + pub start: i64, + pub limit: i64, } pub trait Cursorize: Sized { fn cursor( @@ -25,33 +26,6 @@ pub trait Cursorize: Sized { ) -> CursorPosition; } -pub fn project_to_nodes( - rx: I, - start: i64, - limit: i64, -) -> async_graphql::connection::Connection -where - T: OutputType, - I: IntoIterator, -{ - let rx = Vec::from_iter(rx); - let mut gql = async_graphql::connection::Connection::new( - rx.first().map(|(_, _total)| start > 0).unwrap_or(false), - rx.first().map(|(_, total)| start + limit < *total).unwrap_or(false), - ); - - gql.edges.append( - &mut rx - .into_iter() - .enumerate() - .map(|(pos, (agent, _count))| { - Edge::with_additional_fields((pos as i32) + (start as i32), agent, EmptyFields) - }) - .collect(), - ); - gql -} - impl Cursorize for T { fn cursor( self, diff --git a/crates/api/src/persistence/database.rs b/crates/chronicle-persistence/src/database.rs similarity index 97% rename from crates/api/src/persistence/database.rs rename to crates/chronicle-persistence/src/database.rs index 97e505c25..5f700cac7 100644 --- a/crates/api/src/persistence/database.rs +++ b/crates/chronicle-persistence/src/database.rs @@ -1,6 +1,6 @@ use diesel::{r2d2::ConnectionManager, PgConnection}; -use r2d2::Pool; +use diesel::r2d2::Pool; use std::{fmt::Display, time::Duration}; #[async_trait::async_trait] diff --git a/crates/api/src/persistence/mod.rs b/crates/chronicle-persistence/src/lib.rs similarity index 96% rename from crates/api/src/persistence/mod.rs rename to crates/chronicle-persistence/src/lib.rs index 81d9730a9..014983981 100644 --- a/crates/api/src/persistence/mod.rs +++ b/crates/chronicle-persistence/src/lib.rs @@ -18,29 +18,47 @@ use diesel::{ PgConnection, }; use diesel_migrations::{embed_migrations, EmbeddedMigrations}; -use protocol_substrate_chronicle::protocol::{BlockId, BlockIdError}; +use protocol_substrate_chronicle::protocol::BlockId; use thiserror::Error; use tracing::{debug, instrument, warn}; use uuid::Uuid; pub mod database; -mod query; -pub(crate) mod schema; +pub mod cursor; +pub mod query; +pub mod queryable; +pub mod schema; pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); #[derive(Error, Debug)] pub enum StoreError { #[error("Database operation failed: {0}")] - Db(#[from] diesel::result::Error), + Db( + #[from] + #[source] + diesel::result::Error, + ), #[error("Database connection failed (maybe check PGPASSWORD): {0}")] - DbConnection(#[from] diesel::ConnectionError), + DbConnection( + #[from] + #[source] + diesel::ConnectionError, + ), #[error("Database migration failed: {0}")] - DbMigration(#[from] Box), + DbMigration( + #[from] + #[source] + Box, + ), #[error("Connection pool error: {0}")] - DbPool(#[from] r2d2::Error), + DbPool( + #[from] + #[source] + r2d2::Error, + ), #[error("Infallible")] Infallible(#[from] std::convert::Infallible), @@ -54,19 +72,35 @@ pub enum StoreError { InvalidNamespace(NamespaceId), #[error("Unreadable Attribute: {0}")] - Json(#[from] serde_json::Error), + Json( + #[from] + #[source] + serde_json::Error, + ), #[error("Parse blockid: {0}")] - ParseBlockId(#[from] BlockIdError), + ParseBlockId( + #[from] + #[source] + protocol_substrate_chronicle::protocol::BlockIdError, + ), #[error("Invalid transaction ID: {0}")] - TransactionId(#[from] ChronicleTransactionIdError), + TransactionId( + #[from] + #[source] + ChronicleTransactionIdError, + ), #[error("Could not locate record in store")] RecordNotFound, #[error("Invalid UUID: {0}")] - Uuid(#[from] uuid::Error), + Uuid( + #[from] + #[source] + uuid::Error, + ), } #[derive(Debug)] @@ -76,13 +110,6 @@ pub struct ConnectionOptions { pub busy_timeout: Option, } -#[instrument] -fn sleeper(attempts: i32) -> bool { - warn!(attempts, "SQLITE_BUSY, retrying"); - std::thread::sleep(std::time::Duration::from_millis(250)); - true -} - #[derive(Derivative)] #[derivative(Debug, Clone)] pub struct Store { @@ -92,11 +119,7 @@ pub struct Store { impl Store { #[instrument(name = "Bind namespace", skip(self))] - pub(crate) fn namespace_binding( - &self, - external_id: &str, - uuid: Uuid, - ) -> Result<(), StoreError> { + pub fn namespace_binding(&self, external_id: &str, uuid: Uuid) -> Result<(), StoreError> { use schema::namespace::dsl; let uuid = uuid.to_string(); @@ -145,7 +168,7 @@ impl Store { } /// Fetch the agent record for the IRI - pub(crate) fn agent_by_agent_external_id_and_namespace( + pub fn agent_by_agent_external_id_and_namespace( &self, connection: &mut PgConnection, external_id: &ExternalId, @@ -420,7 +443,7 @@ impl Store { Ok(()) } - pub(crate) fn apply_prov(&self, prov: &ProvModel) -> Result<(), StoreError> { + pub fn apply_prov(&self, prov: &ProvModel) -> Result<(), StoreError> { self.connection()? .build_transaction() .run(|connection| self.apply_model(connection, prov))?; @@ -682,14 +705,14 @@ impl Store { Ok(()) } - pub(crate) fn connection( + pub fn connection( &self, ) -> Result>, StoreError> { - Ok(self.pool.get()?) + self.pool.get().map_err(StoreError::DbPool) } #[instrument(skip(connection))] - pub(crate) fn get_current_agent( + pub fn get_current_agent( &self, connection: &mut PgConnection, ) -> Result { @@ -701,7 +724,7 @@ impl Store { /// Get the last fully synchronized offset #[instrument] - pub(crate) fn get_last_block_id(&self) -> Result, StoreError> { + pub fn get_last_block_id(&self) -> Result, StoreError> { use schema::ledgersync::dsl; self.connection()?.build_transaction().run(|connection| { let block_id_and_tx = schema::ledgersync::table @@ -719,7 +742,7 @@ impl Store { } #[instrument(skip(connection))] - pub(crate) fn namespace_by_external_id( + pub fn namespace_by_external_id( &self, connection: &mut PgConnection, namespace: &ExternalId, @@ -737,11 +760,11 @@ impl Store { } #[instrument] - pub(crate) fn new(pool: Pool>) -> Result { + pub fn new(pool: Pool>) -> Result { Ok(Store { pool }) } - pub(crate) fn prov_model_for_agent( + pub fn prov_model_for_agent( &self, agent: query::Agent, namespaceid: &NamespaceId, @@ -815,7 +838,7 @@ impl Store { Ok(()) } - pub(crate) fn prov_model_for_activity( + pub fn prov_model_for_activity( &self, activity: query::Activity, namespaceid: &NamespaceId, @@ -909,7 +932,7 @@ impl Store { Ok(()) } - pub(crate) fn prov_model_for_entity( + pub fn prov_model_for_entity( &self, entity: query::Entity, namespace_id: &NamespaceId, @@ -1005,7 +1028,7 @@ impl Store { } #[instrument(skip(connection))] - pub(crate) fn prov_model_for_namespace( + pub fn prov_model_for_namespace( &self, connection: &mut PgConnection, namespace: &NamespaceId, @@ -1043,7 +1066,7 @@ impl Store { /// Set the last fully synchronized offset #[instrument] - pub(crate) fn set_last_block_id( + pub fn set_last_block_id( &self, block_id: &BlockId, tx_id: ChronicleTransactionId, @@ -1066,7 +1089,7 @@ impl Store { } #[instrument(skip(connection))] - pub(crate) fn use_agent( + pub fn use_agent( &self, connection: &mut PgConnection, external_id: &ExternalId, @@ -1218,7 +1241,7 @@ impl Store { Ok(model) } - pub(crate) fn prov_model_for_usage( + pub fn prov_model_for_usage( &self, connection: &mut PgConnection, mut model: ProvModel, diff --git a/crates/api/src/persistence/query.rs b/crates/chronicle-persistence/src/query.rs similarity index 95% rename from crates/api/src/persistence/query.rs rename to crates/chronicle-persistence/src/query.rs index b4dc561eb..e779ee13c 100644 --- a/crates/api/src/persistence/query.rs +++ b/crates/chronicle-persistence/src/query.rs @@ -72,14 +72,6 @@ pub struct Agent { pub current: i32, pub identity_id: Option, } - -#[derive(Debug, Queryable)] -pub struct Identity { - pub id: i32, - pub namespace_id: i32, - pub public_key: String, -} - #[derive(Debug, Queryable, Selectable)] #[diesel(table_name = activity)] pub struct Activity { diff --git a/crates/chronicle-persistence/src/queryable.rs b/crates/chronicle-persistence/src/queryable.rs new file mode 100644 index 000000000..9dfc97806 --- /dev/null +++ b/crates/chronicle-persistence/src/queryable.rs @@ -0,0 +1,57 @@ +use async_graphql::{Object, SimpleObject}; +use chrono::NaiveDateTime; +use diesel::{Queryable, Selectable}; + +#[derive(Default, Queryable, Selectable, SimpleObject)] +#[diesel(table_name = crate::schema::agent)] +pub struct Agent { + pub id: i32, + pub external_id: String, + pub namespace_id: i32, + pub domaintype: Option, + pub current: i32, + pub identity_id: Option, +} + +#[derive(Default, Queryable, Selectable, SimpleObject)] +#[diesel(table_name = crate::schema::activity)] +pub struct Activity { + pub id: i32, + pub external_id: String, + pub namespace_id: i32, + pub domaintype: Option, + pub started: Option, + pub ended: Option, +} + +#[derive(Queryable, Selectable, SimpleObject)] +#[diesel(table_name = crate::schema::entity)] +pub struct Entity { + pub id: i32, + pub external_id: String, + pub namespace_id: i32, + pub domaintype: Option, +} + +#[derive(Default, Queryable)] +pub struct Namespace { + _id: i32, + uuid: String, + external_id: String, +} + +#[Object] +/// # `chronicle:Namespace` +/// +/// An IRI containing an external id and uuid part, used for disambiguation. +/// In order to work on the same namespace discrete Chronicle instances must share +/// the uuid part. +impl Namespace { + async fn external_id(&self) -> &str { + &self.external_id + } + + async fn uuid(&self) -> &str { + &self.uuid + } +} diff --git a/crates/api/src/persistence/schema.rs b/crates/chronicle-persistence/src/schema.rs similarity index 100% rename from crates/api/src/persistence/schema.rs rename to crates/chronicle-persistence/src/schema.rs diff --git a/crates/chronicle-signing/src/error.rs b/crates/chronicle-signing/src/error.rs index 018af22c0..76f74e4aa 100644 --- a/crates/chronicle-signing/src/error.rs +++ b/crates/chronicle-signing/src/error.rs @@ -14,6 +14,7 @@ pub enum SecretError { #[error("Vault {source}")] SecretVault { #[from] + #[source] source: anyhow::Error, }, } diff --git a/crates/chronicle-signing/src/lib.rs b/crates/chronicle-signing/src/lib.rs index ca9c864dd..1fb875d57 100644 --- a/crates/chronicle-signing/src/lib.rs +++ b/crates/chronicle-signing/src/lib.rs @@ -41,6 +41,7 @@ pub enum SecretError { #[error("Vault {source}")] SecretVault { #[from] + #[source] source: SecretVaultError, }, diff --git a/crates/chronicle-synth/Cargo.toml b/crates/chronicle-synth/Cargo.toml index f00d6b7ab..fa747ab6f 100644 --- a/crates/chronicle-synth/Cargo.toml +++ b/crates/chronicle-synth/Cargo.toml @@ -13,7 +13,6 @@ name = "chronicle-synth" path = "src/generate.rs" [dependencies] -chronicle = { path = "../chronicle" } clap = { workspace = true } maplit = { workspace = true } owo-colors = { workspace = true } @@ -22,6 +21,11 @@ serde_json = { workspace = true } serde_yaml = { workspace = true } thiserror = { workspace = true } +#local dependencies + +chronicle = { path = "../chronicle" } +common = { path = "../common", features = ["std"] } + [dev-dependencies] assert_fs = { workspace = true } insta = { workspace = true, features = ["json"] } diff --git a/crates/chronicle-synth/src/domain.rs b/crates/chronicle-synth/src/domain.rs index 847fec857..f369b752f 100644 --- a/crates/chronicle-synth/src/domain.rs +++ b/crates/chronicle-synth/src/domain.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, path::Path}; -use chronicle::codegen::model::PrimitiveType; +use common::domain::PrimitiveType; use serde::Deserialize; use serde_json::json; @@ -49,7 +49,7 @@ impl TypesAttributesRoles { Ok(domain) } } - + #[allow(non_local_definitions)] impl From for TypesAttributesRoles { fn from(value: ChronicleDomain) -> Self { let mut attribute_types = BTreeMap::new(); diff --git a/crates/chronicle-synth/src/error.rs b/crates/chronicle-synth/src/error.rs index a0cc8a833..fba455705 100644 --- a/crates/chronicle-synth/src/error.rs +++ b/crates/chronicle-synth/src/error.rs @@ -3,14 +3,30 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum ChronicleSynthError { #[error("Chronicle domain parsing error: {0}")] - ModelError(#[from] chronicle::codegen::model::ModelError), + ModelError( + #[from] + #[source] + common::domain::ModelError, + ), #[error("Invalid JSON: {0}")] - JsonError(#[from] serde_json::Error), + JsonError( + #[from] + #[source] + serde_json::Error, + ), #[error("I/O error: {0}")] - IO(#[from] std::io::Error), + IO( + #[from] + #[source] + std::io::Error, + ), #[error("YAML parsing error: {0}")] - YamlError(#[from] serde_yaml::Error), + YamlError( + #[from] + #[source] + serde_yaml::Error, + ), } diff --git a/crates/chronicle-test-infrastructure/Cargo.toml b/crates/chronicle-test-infrastructure/Cargo.toml index b11df0e08..aa247c531 100644 --- a/crates/chronicle-test-infrastructure/Cargo.toml +++ b/crates/chronicle-test-infrastructure/Cargo.toml @@ -31,7 +31,7 @@ portpicker = { workspace = true } protocol-abstract = { path = "../protocol-abstract" } protocol-substrate = { path = "../protocol-substrate" } protocol-substrate-chronicle = { path = "../protocol-substrate-chronicle" } -r2d2 = { workspace = true } +r2d2 = { version = "^0.8.1" } scale-info = { version = "2.10.0", default-features = false, features = [ "derive", ] } diff --git a/crates/chronicle-test-infrastructure/src/api_test.rs b/crates/chronicle-test-infrastructure/src/api_test.rs index d7e30ea9c..7c84e50fe 100644 --- a/crates/chronicle-test-infrastructure/src/api_test.rs +++ b/crates/chronicle-test-infrastructure/src/api_test.rs @@ -188,7 +188,7 @@ async fn create_namespace() { insta::assert_json_snapshot!(api .dispatch(ApiCommand::NameSpace(NamespaceCommand::Create { - external_id: "testns".into(), + id: "testns".into(), }), identity) .await .unwrap() @@ -214,7 +214,7 @@ async fn create_agent() { let identity = AuthId::chronicle(); insta::assert_json_snapshot!(api.dispatch(ApiCommand::Agent(AgentCommand::Create { - external_id: "testagent".into(), + id: "testagent".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -269,7 +269,7 @@ async fn create_system_activity() { let identity = AuthId::chronicle(); insta::assert_json_snapshot!(api.dispatch(ApiCommand::Activity(ActivityCommand::Create { - external_id: "testactivity".into(), + id: "testactivity".into(), namespace: common::prov::SYSTEM_ID.into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -315,7 +315,7 @@ async fn create_activity() { let identity = AuthId::chronicle(); insta::assert_json_snapshot!(api.dispatch(ApiCommand::Activity(ActivityCommand::Create { - external_id: "testactivity".into(), + id: "testactivity".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -371,7 +371,7 @@ async fn start_activity() { insta::assert_json_snapshot!( api.dispatch(ApiCommand::Agent(AgentCommand::Create { - external_id: "testagent".into(), + id: "testagent".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -482,7 +482,7 @@ async fn contradict_attributes() { insta::assert_json_snapshot!( api.dispatch(ApiCommand::Agent(AgentCommand::Create { - external_id: "testagent".into(), + id: "testagent".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -532,7 +532,7 @@ async fn contradict_attributes() { let res = api .dispatch( ApiCommand::Agent(AgentCommand::Create { - external_id: "testagent".into(), + id: "testagent".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -562,7 +562,7 @@ async fn contradict_start_time() { insta::assert_json_snapshot!( api.dispatch(ApiCommand::Agent(AgentCommand::Create { - external_id: "testagent".into(), + id: "testagent".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -688,7 +688,7 @@ async fn contradict_end_time() { insta::assert_json_snapshot!( api.dispatch(ApiCommand::Agent(AgentCommand::Create { - external_id: "testagent".into(), + id: "testagent".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -814,7 +814,7 @@ async fn end_activity() { insta::assert_json_snapshot!( api.dispatch(ApiCommand::Agent(AgentCommand::Create { - external_id: "testagent".into(), + id: "testagent".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -972,7 +972,7 @@ async fn activity_use() { insta::assert_json_snapshot!( api.dispatch(ApiCommand::Agent(AgentCommand::Create { - external_id: "testagent".into(), + id: "testagent".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -1031,7 +1031,7 @@ async fn activity_use() { insta::assert_json_snapshot!( api.dispatch(ApiCommand::Activity(ActivityCommand::Create { - external_id: "testactivity".into(), + id: "testactivity".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), @@ -1174,7 +1174,7 @@ async fn activity_generate() { insta::assert_json_snapshot!( api.dispatch(ApiCommand::Activity(ActivityCommand::Create { - external_id: "testactivity".into(), + id: "testactivity".into(), namespace: "testns".into(), attributes: Attributes::new( Some(DomaintypeId::from_external_id("test")), diff --git a/crates/chronicle-test-infrastructure/src/substitutes/mod.rs b/crates/chronicle-test-infrastructure/src/substitutes/mod.rs index 08eb24d1e..e32326687 100644 --- a/crates/chronicle-test-infrastructure/src/substitutes/mod.rs +++ b/crates/chronicle-test-infrastructure/src/substitutes/mod.rs @@ -17,8 +17,10 @@ use chronicle_signing::{ CHRONICLE_NAMESPACE, }; -use diesel::{r2d2::ConnectionManager, Connection, PgConnection}; -use r2d2::Pool; +use diesel::{ + r2d2::{ConnectionManager, Pool}, + Connection, PgConnection, +}; use testcontainers::{images::postgres::Postgres, Container}; use lazy_static::lazy_static; @@ -67,10 +69,22 @@ impl<'a> Default for TemporaryDatabase<'a> { pub struct TestDispatch<'a> { api: ApiDispatch, - _db: TemporaryDatabase<'a>, + db: TemporaryDatabase<'a>, _substrate: Stubstrate, } +impl<'a> TestDispatch<'a> { + /// Returns a reference to the ApiDispatch. + pub fn api_dispatch(&self) -> &ApiDispatch { + &self.api + } + + /// Returns a reference to the TemporaryDatabase. + pub fn temporary_database(&self) -> &TemporaryDatabase<'a> { + &self.db + } +} + impl<'a> TestDispatch<'a> { pub async fn dispatch( &mut self, @@ -85,8 +99,9 @@ impl<'a> TestDispatch<'a> { let commit = self.api.notify_commit.subscribe().recv().await.unwrap(); match commit { common::ledger::SubmissionStage::Submitted(Ok(_)) => continue, - common::ledger::SubmissionStage::Committed(commit, _id) => - return Ok(Some((commit.delta, commit.tx_id))), + common::ledger::SubmissionStage::Committed(commit, _id) => { + return Ok(Some((commit.delta, commit.tx_id))) + }, common::ledger::SubmissionStage::Submitted(Err(e)) => panic!("{e:?}"), common::ledger::SubmissionStage::NotCommitted((_, tx, _id)) => { panic!("{tx:?}") @@ -94,8 +109,9 @@ impl<'a> TestDispatch<'a> { } } }, - ApiResponse::AlreadyRecorded { subject: _, prov } => - Ok(Some((prov, ChronicleTransactionId::default()))), + ApiResponse::AlreadyRecorded { subject: _, prov } => { + Ok(Some((prov, ChronicleTransactionId::default()))) + }, _ => Ok(None), } } @@ -147,7 +163,7 @@ pub async fn test_api<'a>() -> TestDispatch<'a> { TestDispatch { api: dispatch, - _db: database, // share the lifetime + db: database, // share the lifetime _substrate: embed_substrate, } } diff --git a/crates/chronicle-test-infrastructure/src/substitutes/stubstrate.rs b/crates/chronicle-test-infrastructure/src/substitutes/stubstrate.rs index 8ea9fff70..064f6aa11 100644 --- a/crates/chronicle-test-infrastructure/src/substitutes/stubstrate.rs +++ b/crates/chronicle-test-infrastructure/src/substitutes/stubstrate.rs @@ -113,16 +113,14 @@ impl LedgerWriter for Stubstrate { let opa_event = match ev { RuntimeEvent::ChronicleModule(event) => match event { - Event::::Applied(diff, identity, correlation_id) => { - Some(ChronicleEvent::Committed { diff, identity, correlation_id }) - }, - Event::::Contradiction(contradiction, identity, correlation_id) => { + Event::::Applied(diff, identity, correlation_id) => + Some(ChronicleEvent::Committed { diff, identity, correlation_id }), + Event::::Contradiction(contradiction, identity, correlation_id) => Some(ChronicleEvent::Contradicted { contradiction, identity, correlation_id, - }) - }, + }), _ => None, }, _ => None, diff --git a/crates/chronicle/Cargo.toml b/crates/chronicle/Cargo.toml index 6735ed2dc..6a82594af 100644 --- a/crates/chronicle/Cargo.toml +++ b/crates/chronicle/Cargo.toml @@ -8,26 +8,17 @@ version = "0.7.5" [dependencies] Inflector = { workspace = true } -api = { path = "../api" } async-graphql = { workspace = true } async-trait = { workspace = true } cfg-if = { workspace = true } chronicle-signing = { workspace = true } -chronicle-telemetry = { path = "../chronicle-telemetry" } chrono = { workspace = true, features = ["serde"] } clap = { workspace = true, features = ["derive", "env"] } clap_complete = { workspace = true } colored_json = { workspace = true } -common = { path = "../common", features = [ - "json-ld", - "std", - "parity-encoding", - "graphql-bindings", -] } const_format = { workspace = true } diesel = { workspace = true } dotenvy = { workspace = true } -embedded-substrate = { path = "../embedded-substrate", optional = true } futures = { workspace = true } genco = { workspace = true } hex = { workspace = true } @@ -39,9 +30,6 @@ jsonschema = { workspace = true } opa = { workspace = true } opentelemetry = { workspace = true } percent-encoding = { workspace = true } -protocol-substrate = { path = "../protocol-substrate" } -protocol-substrate-chronicle = { path = "../protocol-substrate-chronicle" } -protocol-substrate-opa = { path = "../protocol-substrate-opa" } question = { workspace = true } rand = { workspace = true } rand_core = { workspace = true } @@ -60,6 +48,22 @@ user-error = { workspace = true } uuid = { workspace = true } valico = { workspace = true } +#Local dependencies +api = { path = "../api" } +chronicle-arrow = { path = "../chronicle-arrow" } +chronicle-persistence = { path = "../chronicle-persistence" } +chronicle-telemetry = { path = "../chronicle-telemetry" } +common = { path = "../common", features = [ + "json-ld", + "std", + "parity-encoding", + "graphql-bindings", +] } +embedded-substrate = { path = "../embedded-substrate", optional = true } +protocol-substrate = { path = "../protocol-substrate" } +protocol-substrate-chronicle = { path = "../protocol-substrate-chronicle" } +protocol-substrate-opa = { path = "../protocol-substrate-opa" } + [features] devmode = ["dep:embedded-substrate"] strict = [] diff --git a/crates/chronicle/src/bootstrap/cli.rs b/crates/chronicle/src/bootstrap/cli.rs index 9ea4f2ec1..24ddf205b 100644 --- a/crates/chronicle/src/bootstrap/cli.rs +++ b/crates/chronicle/src/bootstrap/cli.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, convert::Infallible}; +use std::convert::Infallible; use api::{ commands::{ActivityCommand, AgentCommand, ApiCommand, EntityCommand}, @@ -39,64 +39,129 @@ pub enum CliError { InvalidArgument { arg: String, expected: String, got: String }, #[error("Bad argument: {0}")] - ArgumentParsing(#[from] clap::Error), + ArgumentParsing( + #[from] + #[source] + clap::Error, + ), #[error("Invalid IRI: {0}")] - InvalidIri(#[from] iri_string::validate::Error), + InvalidIri( + #[from] + #[source] + iri_string::validate::Error, + ), #[error("Invalid Chronicle IRI: {0}")] - InvalidChronicleIri(#[from] ParseIriError), + InvalidChronicleIri( + #[from] + #[source] + ParseIriError, + ), #[error("Invalid JSON: {0}")] - InvalidJson(#[from] serde_json::Error), + InvalidJson( + #[from] + #[source] + serde_json::Error, + ), #[error("Invalid URI: {0}")] - InvalidUri(#[from] url::ParseError), + InvalidUri( + #[from] + #[source] + url::ParseError, + ), #[error("Invalid timestamp: {0}")] - InvalidTimestamp(#[from] chrono::ParseError), + InvalidTimestamp( + #[from] + #[source] + chrono::ParseError, + ), #[error("Invalid coercion: {arg}")] InvalidCoercion { arg: String }, #[error("API failure: {0}")] - ApiError(#[from] ApiError), + ApiError( + #[from] + #[source] + ApiError, + ), #[error("Secrets : {0}")] - Secrets(#[from] SecretError), + Secrets( + #[from] + #[source] + SecretError, + ), #[error("IO error: {0}")] - InputOutput(#[from] std::io::Error), + InputOutput( + #[from] + #[source] + std::io::Error, + ), #[error("Invalid configuration file: {0}")] - ConfigInvalid(#[from] toml::de::Error), + ConfigInvalid( + #[from] + #[source] + toml::de::Error, + ), #[error("Invalid path: {path}")] InvalidPath { path: String }, #[error("Invalid JSON-LD: {0}")] - Ld(#[from] CompactionError), + Ld( + #[from] + #[source] + CompactionError, + ), #[error("Failure in commit notification stream: {0}")] - CommitNoticiationStream(#[from] RecvError), + CommitNoticiationStream( + #[from] + #[source] + RecvError, + ), #[error("Policy loader error: {0}")] - OpaPolicyLoader(#[from] PolicyLoaderError), + OpaPolicyLoader( + #[from] + #[source] + PolicyLoaderError, + ), #[error("OPA executor error: {0}")] - OpaExecutor(#[from] OpaExecutorError), + OpaExecutor( + #[from] + #[source] + OpaExecutorError, + ), #[error("Sawtooth communication error: {source}")] SubstrateError { #[from] + #[source] source: SubxtClientError, }, #[error("UTF-8 error: {0}")] - Utf8Error(#[from] std::str::Utf8Error), + Utf8Error( + #[from] + #[source] + std::str::Utf8Error, + ), #[error("Url conversion: {0}")] - FromUrlError(#[from] FromUrlError), + FromUrlError( + #[from] + #[source] + FromUrlError, + ), #[error("No on chain settings, but they are required by Chronicle")] NoOnChainSettings, @@ -356,7 +421,7 @@ impl SubCommand for AgentCliModel { fn matches(&self, matches: &ArgMatches) -> Result, CliError> { if let Some(matches) = matches.subcommand_matches("define") { return Ok(Some(ApiCommand::Agent(AgentCommand::Create { - external_id: name_from::(matches, "external_id", "id")?, + id: name_from::(matches, "external_id", "id")?, namespace: namespace_from(matches)?, attributes: attributes_from(matches, &self.agent.external_id, &self.attributes)?, }))); @@ -568,7 +633,7 @@ impl SubCommand for ActivityCliModel { fn matches(&self, matches: &ArgMatches) -> Result, CliError> { if let Some(matches) = matches.subcommand_matches("define") { return Ok(Some(ApiCommand::Activity(ActivityCommand::Create { - external_id: name_from::(matches, "external_id", "id")?, + id: name_from::(matches, "external_id", "id")?, namespace: namespace_from(matches)?, attributes: attributes_from(matches, &self.activity.external_id, &self.attributes)?, }))); @@ -724,7 +789,7 @@ impl SubCommand for EntityCliModel { fn matches(&self, matches: &ArgMatches) -> Result, CliError> { if let Some(matches) = matches.subcommand_matches("define") { return Ok(Some(ApiCommand::Entity(EntityCommand::Create { - external_id: name_from::(matches, "external_id", "id")?, + id: name_from::(matches, "external_id", "id")?, namespace: namespace_from(matches)?, attributes: attributes_from(matches, &self.entity.external_id, &self.attributes)?, }))); @@ -875,6 +940,15 @@ impl SubCommand for CliModel { Command::new("serve-api") .alias("serve-graphql") .about("Start an API server") + .arg( + Arg::new("arrow-interface") + .long("arrow-interface") + .takes_value(true) + .min_values(1) + .default_values(&["localhost:9983"]) + .env("ARROW_LISTEN_SOCKET") + .help("The arrow flight address"), + ) .arg( Arg::new("interface") .long("interface") diff --git a/crates/chronicle/src/bootstrap/mod.rs b/crates/chronicle/src/bootstrap/mod.rs index 4423ce323..32bd8be4e 100644 --- a/crates/chronicle/src/bootstrap/mod.rs +++ b/crates/chronicle/src/bootstrap/mod.rs @@ -3,10 +3,10 @@ pub mod opa; use api::{ chronicle_graphql::{ChronicleApiServer, ChronicleGraphQl, JwksUri, SecurityConf, UserInfoUri}, commands::ApiResponse, - database::{get_connection_with_retry, DatabaseConnector}, Api, ApiDispatch, ApiError, StoreError, UuidGen, }; -use async_graphql::{async_trait, ObjectType}; +use async_graphql::ObjectType; +use chronicle_persistence::database::{get_connection_with_retry, DatabaseConnector}; use common::{ opa::{ std::{load_bytes_from_stdin, load_bytes_from_url}, @@ -527,8 +527,6 @@ where Ok((ApiResponse::Unit, ret_api)) } else if let Some(matches) = matches.subcommand_matches("import") { - let namespace = get_namespace(matches); - let data = if let Some(url) = matches.value_of("url") { let data = load_bytes_from_url(url).await?; info!("Loaded import data from {:?}", url); @@ -557,18 +555,14 @@ where let op = ChronicleOperation::from_json(&value) .await .expect("Failed to parse imported JSON-LD to ChronicleOperation"); - // Only import operations for the specified namespace - if op.namespace() == &namespace { - operations.push(op); - } + operations.push(op); } info!("Loading import data complete"); let identity = AuthId::chronicle(); - info!("Importing data as root to Chronicle namespace: {namespace}"); - let response = api.handle_import_command(identity, namespace, operations).await?; + let response = api.handle_import_command(identity, operations).await?; Ok((response, ret_api)) } else if let Some(cmd) = cli.matches(&matches)? { @@ -664,6 +658,9 @@ where .to_colored_json_auto() .unwrap() ); + } + (ApiResponse::AlreadyRecordedAll, _api) => { + println!("Import will not result in any data changes"); } (ApiResponse::ImportSubmitted { prov, tx_id }, api) => { let mut tx_notifications = api.notify_commit.subscribe(); diff --git a/crates/chronicle/src/codegen/linter.rs b/crates/chronicle/src/codegen/linter.rs index 061e013c5..069c0701b 100644 --- a/crates/chronicle/src/codegen/linter.rs +++ b/crates/chronicle/src/codegen/linter.rs @@ -1,4 +1,4 @@ -use crate::codegen::model; +use common::domain::{DomainFileInput, ResourceDef}; use jsonschema::{error::ValidationErrorKind, JSONSchema}; use std::{collections::HashSet, path::Path, process::exit}; @@ -70,7 +70,7 @@ fn check_yaml_valid(json_validator: &JSONSchema, yaml_data: &str) { check_json_valid(json_validator, &json_data); } -fn read_json_domain(data: &str) -> model::DomainFileInput { +fn read_json_domain(data: &str) -> DomainFileInput { match serde_json::from_str(data) { Ok(domain) => domain, Err(error) => { @@ -80,7 +80,7 @@ fn read_json_domain(data: &str) -> model::DomainFileInput { } } -fn read_yaml_domain(data: &str) -> model::DomainFileInput { +fn read_yaml_domain(data: &str) -> DomainFileInput { match serde_yaml::from_str(data) { Ok(domain) => domain, Err(error) => { @@ -93,7 +93,7 @@ fn read_yaml_domain(data: &str) -> model::DomainFileInput { fn check_domain_attributes( element: &str, attributes: &HashSet, - named_resources: Vec<(&String, &model::ResourceDef)>, + named_resources: Vec<(&String, &ResourceDef)>, ) { let mut is_error = false; for (name, resource) in named_resources { @@ -109,7 +109,7 @@ fn check_domain_attributes( } } -fn check_domain(domain: model::DomainFileInput) { +fn check_domain(domain: DomainFileInput) { let attributes = domain.attributes.keys().map(std::clone::Clone::clone).collect(); check_domain_attributes("agent", &attributes, domain.agents.iter().collect()); check_domain_attributes("entity", &attributes, domain.entities.iter().collect()); diff --git a/crates/chronicle/src/codegen/mod.rs b/crates/chronicle/src/codegen/mod.rs index d536db9fe..baf3b5a94 100644 --- a/crates/chronicle/src/codegen/mod.rs +++ b/crates/chronicle/src/codegen/mod.rs @@ -1,13 +1,12 @@ #![allow(dead_code)] pub mod linter; -pub mod model; use std::{io::Write, path::Path}; use genco::prelude::*; -pub use model::{AttributesTypeName, Builder, CliName, PrimitiveType, Property, TypeName}; +pub use common::domain::{AttributesTypeName, Builder, CliName, PrimitiveType, Property, TypeName}; -pub use self::model::{ActivityDef, AgentDef, AttributeDef, ChronicleDomainDef, EntityDef}; +pub use common::domain::{ActivityDef, AgentDef, AttributeDef, ChronicleDomainDef, EntityDef}; fn agent_union_type_name() -> String { "Agent".to_owned() @@ -308,9 +307,9 @@ fn gen_activity_union(activities: &[ActivityDef]) -> rust::Tokens { fn gen_activity_definition(activity: &ActivityDef) -> rust::Tokens { let abstract_activity = - &rust::import("chronicle::api::chronicle_graphql", "Activity").qualified(); + &rust::import("chronicle::persistence::queryable", "Activity").qualified(); let activity_impl = &rust::import("chronicle::api::chronicle_graphql", "activity").qualified(); - let namespace = &rust::import("chronicle::api::chronicle_graphql", "Namespace").qualified(); + let namespace = &rust::import("chronicle::persistence::queryable", "Namespace").qualified(); let activity_id = &rust::import("chronicle::common::prov", "ActivityId").qualified(); let async_graphql_error_extensions = &rust::import("chronicle::async_graphql", "ErrorExtensions").qualified(); @@ -463,9 +462,9 @@ fn gen_activity_definition(activity: &ActivityDef) -> rust::Tokens { } fn gen_entity_definition(entity: &EntityDef) -> rust::Tokens { - let abstract_entity = &rust::import("chronicle::api::chronicle_graphql", "Entity").qualified(); + let abstract_entity = &rust::import("chronicle::persistence::queryable", "Entity").qualified(); let entity_impl = &rust::import("chronicle::api::chronicle_graphql", "entity").qualified(); - let namespace = &rust::import("chronicle::api::chronicle_graphql", "Namespace").qualified(); + let namespace = &rust::import("chronicle::persistence::queryable", "Namespace").qualified(); let entity_id = &rust::import("chronicle::common::prov", "EntityId").qualified(); let object = rust::import("chronicle::async_graphql", "Object").qualified(); @@ -630,10 +629,9 @@ fn gen_entity_definition(entity: &EntityDef) -> rust::Tokens { } fn gen_agent_definition(agent: &AgentDef) -> rust::Tokens { - let abstract_agent = &rust::import("chronicle::api::chronicle_graphql", "Agent").qualified(); + let abstract_agent = &rust::import("chronicle::persistence::queryable", "Agent").qualified(); let agent_impl = &rust::import("chronicle::api::chronicle_graphql", "agent").qualified(); - let namespace = &rust::import("chronicle::api::chronicle_graphql", "Namespace").qualified(); - let identity = &rust::import("chronicle::api::chronicle_graphql", "Identity").qualified(); + let namespace = &rust::import("chronicle::persistence::queryable", "Namespace").qualified(); let agent_union_type = &agent_union_type_name(); let object = rust::import("chronicle::async_graphql", "Object").qualified(); let async_result = &rust::import("chronicle::async_graphql", "Result").qualified(); @@ -648,7 +646,6 @@ fn gen_agent_definition(agent: &AgentDef) -> rust::Tokens { let attribution_doc = include_str!("../../../../domain_docs/attribution.md"); let external_id_doc = include_str!("../../../../domain_docs/external_id.md"); let id_doc = include_str!("../../../../domain_docs/id.md"); - let identity_doc = include_str!("../../../../domain_docs/identity.md"); let namespace_doc = include_str!("../../../../domain_docs/namespace.md"); let type_doc = include_str!("../../../../domain_docs/type.md"); @@ -679,11 +676,6 @@ fn gen_agent_definition(agent: &AgentDef) -> rust::Tokens { #agent_impl::namespace(self.0.namespace_id, ctx).await.map_err(|e| #async_graphql_error_extensions::extend(&e)) } - #[doc = #_(#identity_doc)] - async fn identity<'a>(&self, ctx: &#context<'a>) -> #async_result> { - #agent_impl::identity(self.0.identity_id, ctx).await.map_err(|e| #async_graphql_error_extensions::extend(&e)) - } - #[doc = #_(#acted_on_behalf_of_doc)] async fn acted_on_behalf_of<'a>(&self, ctx: &#context<'a>) -> #async_result> { Ok(#agent_impl::acted_on_behalf_of(self.0.id, ctx) @@ -856,10 +848,10 @@ fn gen_attribute_definition(typ: impl TypeName, attributes: &[AttributeDef]) -> } fn gen_mappers(domain: &ChronicleDomainDef) -> rust::Tokens { - let agent_impl = &rust::import("chronicle::api::chronicle_graphql", "Agent").qualified(); + let agent_impl = &rust::import("chronicle::persistence::queryable", "Agent").qualified(); let role = &rust::import("chronicle::common::prov", "Role").qualified(); - let entity_impl = &rust::import("chronicle::api::chronicle_graphql", "Entity").qualified(); - let activity_impl = &rust::import("chronicle::api::chronicle_graphql", "Activity").qualified(); + let entity_impl = &rust::import("chronicle::persistence::queryable", "Entity").qualified(); + let activity_impl = &rust::import("chronicle::persistence::queryable", "Activity").qualified(); quote! { #[allow(clippy::match_single_binding)] diff --git a/crates/chronicle/src/lib.rs b/crates/chronicle/src/lib.rs index 7ceff7c48..506d04240 100644 --- a/crates/chronicle/src/lib.rs +++ b/crates/chronicle/src/lib.rs @@ -4,6 +4,7 @@ pub mod codegen; /// Re-export dependencies for generated code pub use api; pub use async_graphql; +pub use chronicle_persistence as persistence; pub use chrono; pub use common; pub use serde_json; diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index f158efa70..577f4c315 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -7,6 +7,7 @@ version = "0.7.5" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +Inflector = { workspace = true, optional = true } anyhow = { version = "^1", default-features = false } async-graphql = { version = "^7", features = [ "opentelemetry", @@ -20,7 +21,7 @@ chrono = { version = "^0.4", default-features = false, features = [ "serde", "alloc", ] } -diesel = { version = "2.0.0-rc.0", features = [ +diesel = { version = "2.1", features = [ "postgres", "uuid", "chrono", @@ -75,7 +76,8 @@ serde = { version = "1.0", default-features = false, features = [ "derive", ] } serde_derive = { version = "1.0", default-features = false } -serde_json = { version = "1.0", default-features = false } +serde_json = { workspace = true, optional = true } +serde_yaml = { workspace = true, optional = true } sp-core = { version = "25.0.0", default-features = false } sp-std = { version = "11.0.0", optional = true } thiserror = { workspace = true } @@ -92,6 +94,7 @@ lazy_static = { workspace = true } serde_json = { workspace = true } [dev-dependencies] +assert_fs = { workspace = true } criterion = { workspace = true } insta = { workspace = true, features = ["json"] } mockito = { workspace = true } @@ -124,6 +127,8 @@ std = [ "scale-decode/std", "scale-info/std", "uuid/v4", + "dep:Inflector", + "dep:serde_yaml", ] # Enable parity support, annoyingly lazy_static has a non standard way of enabling non_std parity-encoding = [ diff --git a/crates/chronicle/src/codegen/model.rs b/crates/common/src/domain.rs similarity index 96% rename from crates/chronicle/src/codegen/model.rs rename to crates/common/src/domain.rs index 426305ca9..865438b7d 100644 --- a/crates/chronicle/src/codegen/model.rs +++ b/crates/common/src/domain.rs @@ -13,13 +13,25 @@ pub enum ModelError { AttributeNotDefined { attr: String }, #[error("Model file not readable: {0}")] - ModelFileNotReadable(#[from] std::io::Error), + ModelFileNotReadable( + #[from] + #[source] + std::io::Error, + ), #[error("Model file invalid JSON: {0}")] - ModelFileInvalidJson(#[from] serde_json::Error), + ModelFileInvalidJson( + #[from] + #[source] + serde_json::Error, + ), #[error("Model file invalid YAML: {0}")] - ModelFileInvalidYaml(#[from] serde_yaml::Error), + ModelFileInvalidYaml( + #[from] + #[source] + serde_yaml::Error, + ), } #[derive(Deserialize, Serialize, Debug, Copy, Clone, PartialEq, Eq)] @@ -33,8 +45,8 @@ pub enum PrimitiveType { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AttributeDef { typ: String, - pub(crate) doc: Option, - pub(crate) primitive_type: PrimitiveType, + pub doc: Option, + pub primitive_type: PrimitiveType, } impl TypeName for AttributeDef { @@ -60,11 +72,11 @@ impl AttributeDef { } } - pub(crate) fn as_property(&self) -> String { + pub fn as_property(&self) -> String { to_snake_case(&format!("{}Attribute", self.typ)) } - pub(crate) fn from_attribute_file_input(external_id: String, attr: AttributeFileInput) -> Self { + pub fn from_attribute_file_input(external_id: String, attr: AttributeFileInput) -> Self { AttributeDef { typ: external_id, doc: attr.doc, primitive_type: attr.typ } } } @@ -128,9 +140,9 @@ where #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AgentDef { - pub(crate) external_id: String, - pub(crate) doc: Option, - pub(crate) attributes: Vec, + pub external_id: String, + pub doc: Option, + pub attributes: Vec, } impl TypeName for &AgentDef { @@ -179,9 +191,9 @@ impl AgentDef { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EntityDef { - pub(crate) external_id: String, - pub(crate) doc: Option, - pub(crate) attributes: Vec, + pub external_id: String, + pub doc: Option, + pub attributes: Vec, } impl TypeName for &EntityDef { @@ -195,7 +207,7 @@ impl TypeName for &EntityDef { } impl EntityDef { - pub(crate) fn new( + pub fn new( external_id: impl AsRef, doc: Option, attributes: Vec, @@ -203,7 +215,7 @@ impl EntityDef { Self { external_id: external_id.as_ref().to_string(), doc, attributes } } - pub(crate) fn from_input<'a>( + pub fn from_input<'a>( external_id: String, doc: Option, attributes: &BTreeMap, @@ -230,9 +242,9 @@ impl EntityDef { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ActivityDef { - pub(crate) external_id: String, - pub(crate) doc: Option, - pub(crate) attributes: Vec, + pub external_id: String, + pub doc: Option, + pub attributes: Vec, } impl TypeName for &ActivityDef { @@ -246,7 +258,7 @@ impl TypeName for &ActivityDef { } impl ActivityDef { - pub(crate) fn new( + pub fn new( external_id: impl AsRef, doc: Option, attributes: Vec, @@ -254,7 +266,7 @@ impl ActivityDef { Self { external_id: external_id.as_ref().to_string(), doc, attributes } } - pub(crate) fn from_input<'a>( + pub fn from_input<'a>( external_id: String, doc: Option, attributes: &BTreeMap, @@ -281,7 +293,7 @@ impl ActivityDef { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RoleDef { - pub(crate) external_id: String, + pub external_id: String, } impl RoleDef { @@ -327,12 +339,12 @@ fn preserve_inflection_for_kind(kind: &str, id: &str) -> String { #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct ChronicleDomainDef { name: String, - pub(crate) attributes: Vec, - pub(crate) agents: Vec, - pub(crate) entities: Vec, - pub(crate) activities: Vec, - pub(crate) roles_doc: Option, - pub(crate) roles: Vec, + pub attributes: Vec, + pub agents: Vec, + pub entities: Vec, + pub activities: Vec, + pub roles_doc: Option, + pub roles: Vec, } pub struct AgentBuilder<'a>(&'a ChronicleDomainDef, AgentDef); @@ -505,8 +517,8 @@ pub struct AttributeRef(pub String); #[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] pub struct ResourceDef { - pub(crate) doc: Option, - pub(crate) attributes: Vec, + pub doc: Option, + pub attributes: Vec, } impl From<&AgentDef> for ResourceDef { @@ -550,17 +562,17 @@ impl From<&ActivityDef> for ResourceDef { #[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Default)] pub struct DomainFileInput { - pub(crate) name: String, - pub(crate) attributes: BTreeMap, - pub(crate) agents: BTreeMap, - pub(crate) entities: BTreeMap, - pub(crate) activities: BTreeMap, - pub(crate) roles_doc: Option, - pub(crate) roles: Vec, + pub name: String, + pub attributes: BTreeMap, + pub agents: BTreeMap, + pub entities: BTreeMap, + pub activities: BTreeMap, + pub roles_doc: Option, + pub roles: Vec, } impl DomainFileInput { - pub(crate) fn new(name: impl AsRef) -> Self { + pub fn new(name: impl AsRef) -> Self { DomainFileInput { name: name.as_ref().to_string(), ..Default::default() } } } @@ -690,7 +702,7 @@ impl ChronicleDomainDef { Ok(builder.build()) } - pub(crate) fn to_json_string(&self) -> Result { + pub fn to_json_string(&self) -> Result { let input: DomainFileInput = self.into(); let json = serde_json::to_string(&input)?; Ok(json) diff --git a/crates/common/src/identity.rs b/crates/common/src/identity.rs index 0667a2bd9..6ae5ca287 100644 --- a/crates/common/src/identity.rs +++ b/crates/common/src/identity.rs @@ -28,10 +28,18 @@ pub enum IdentityError { JwtClaims, #[error("Signer : {0}")] - Signing(#[from] anyhow::Error), + Signing( + #[from] + #[source] + anyhow::Error, + ), #[error("Malformed JSON: {0}")] - SerdeJson(#[from] serde_json::Error), + SerdeJson( + #[from] + #[source] + serde_json::Error, + ), #[error("Serialization error: {0}")] SerdeJsonSerialize(String), diff --git a/crates/common/src/ledger.rs b/crates/common/src/ledger.rs index 2a8d66155..1c3b968dd 100644 --- a/crates/common/src/ledger.rs +++ b/crates/common/src/ledger.rs @@ -475,24 +475,16 @@ impl ChronicleOperation { ChronicleOperation::CreateNamespace(CreateNamespace { id, .. }) => { vec![ChronicleAddress::namespace(id)] }, - ChronicleOperation::AgentExists(AgentExists { namespace, external_id, .. }) => { + ChronicleOperation::AgentExists(AgentExists { namespace, id, .. }) => { vec![ ChronicleAddress::namespace(namespace), - ChronicleAddress::in_namespace( - namespace, - AgentId::from_external_id(external_id), - ), + ChronicleAddress::in_namespace(namespace, id.clone()), ] }, - ChronicleOperation::ActivityExists(ActivityExists { - namespace, external_id, .. - }) => { + ChronicleOperation::ActivityExists(ActivityExists { namespace, id, .. }) => { vec![ ChronicleAddress::namespace(namespace), - ChronicleAddress::in_namespace( - namespace, - ActivityId::from_external_id(external_id), - ), + ChronicleAddress::in_namespace(namespace, id.clone()), ] }, ChronicleOperation::StartActivity(StartActivity { namespace, id, .. }) => { @@ -538,13 +530,10 @@ impl ChronicleOperation { ChronicleAddress::in_namespace(namespace, id.clone()), ] }, - ChronicleOperation::EntityExists(EntityExists { namespace, external_id }) => { + ChronicleOperation::EntityExists(EntityExists { namespace, id }) => { vec![ ChronicleAddress::namespace(namespace), - ChronicleAddress::in_namespace( - namespace, - EntityId::from_external_id(external_id), - ), + ChronicleAddress::in_namespace(namespace, id.clone()), ] }, ChronicleOperation::WasGeneratedBy(WasGeneratedBy { namespace, id, activity }) => vec![ diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 8651e2898..719bea214 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -5,6 +5,8 @@ extern crate serde_derive; pub mod attributes; pub mod context; +#[cfg(feature = "std")] +pub mod domain; pub mod identity; pub mod ledger; pub mod opa; diff --git a/crates/common/src/opa/std.rs b/crates/common/src/opa/std.rs index baeb4d493..cc4aefd9b 100644 --- a/crates/common/src/opa/std.rs +++ b/crates/common/src/opa/std.rs @@ -35,13 +35,21 @@ pub fn key_address(id: impl AsRef) -> KeyAddress { #[derive(Error, Debug)] pub enum FromUrlError { #[error("HTTP error while attempting to read from URL: {0}")] - HTTP(#[from] reqwest::Error), + HTTP( + #[from] + #[source] + reqwest::Error, + ), #[error("Invalid URL scheme: {0}")] InvalidUrlScheme(String), #[error("IO error while attempting to read from URL: {0}")] - IO(#[from] std::io::Error), + IO( + #[from] + #[source] + std::io::Error, + ), } pub enum PathOrUrl { @@ -89,13 +97,25 @@ pub enum OpaExecutorError { AccessDenied, #[error("Identity error: {0}")] - IdentityError(#[from] IdentityError), + IdentityError( + #[from] + #[source] + IdentityError, + ), #[error("Error loading OPA policy: {0}")] - PolicyLoaderError(#[from] PolicyLoaderError), + PolicyLoaderError( + #[from] + #[source] + PolicyLoaderError, + ), #[error("Error evaluating OPA policy: {0}")] - OpaEvaluationError(#[from] anyhow::Error), + OpaEvaluationError( + #[from] + #[source] + anyhow::Error, + ), } #[async_trait::async_trait] @@ -163,13 +183,25 @@ pub enum PolicyLoaderError { MissingPolicy(String), #[error("OPA bundle I/O error: {0}")] - OpaBundleError(#[from] opa::bundle::Error), + OpaBundleError( + #[from] + #[source] + opa::bundle::Error, + ), #[error("Error loading OPA policy: {0}")] - Substrate(#[from] anyhow::Error), + Substrate( + #[from] + #[source] + anyhow::Error, + ), #[error("Error loading from URL: {0}")] - FromUrl(#[from] FromUrlError), + FromUrl( + #[from] + #[source] + FromUrlError, + ), } #[async_trait::async_trait] diff --git a/crates/common/src/protocol.rs b/crates/common/src/protocol.rs index 1578bd152..47dd452b4 100644 --- a/crates/common/src/protocol.rs +++ b/crates/common/src/protocol.rs @@ -4,11 +4,11 @@ use prost::Message; use tracing::span; use crate::{ - identity::SignedIdentity, - prov::{ - operations::ChronicleOperation, to_json_ld::ToJson, ChronicleTransaction, CompactionError, - Contradiction, ExpandedJson, ProcessorError, ProvModel, - }, + identity::SignedIdentity, + prov::{ + operations::ChronicleOperation, to_json_ld::ToJson, ChronicleTransaction, CompactionError, + Contradiction, ExpandedJson, ProcessorError, ProvModel, + }, }; use thiserror::Error; @@ -17,31 +17,36 @@ use self::messages::event::OptionContradiction; #[derive(Error, Debug)] pub enum ProtocolError { - #[error("Protobuf deserialization error {source}")] - ProtobufDeserialize { - #[from] - source: prost::DecodeError, - }, - #[error("Protobuf serialization error {source}")] - ProtobufSerialize { - #[from] - source: prost::EncodeError, - }, - #[error("Serde de/serialization error {source}")] - JsonSerialize { - #[from] - source: serde_json::Error, - }, - #[error("Problem applying delta {source}")] - ProcessorError { - #[from] - source: ProcessorError, - }, - #[error("Could not compact json {source}")] - Compaction { - #[from] - source: CompactionError, - }, + #[error("Protobuf deserialization error {source}")] + ProtobufDeserialize { + #[from] + #[source] + source: prost::DecodeError, + }, + #[error("Protobuf serialization error {source}")] + ProtobufSerialize { + #[from] + #[source] + source: prost::EncodeError, + }, + #[error("Serde de/serialization error {source}")] + JsonSerialize { + #[from] + #[source] + source: serde_json::Error, + }, + #[error("Problem applying delta {source}")] + ProcessorError { + #[from] + #[source] + source: ProcessorError, + }, + #[error("Could not compact json {source}")] + Compaction { + #[from] + #[source] + source: CompactionError, + }, } static PROTOCOL_VERSION: &str = "2"; @@ -49,7 +54,7 @@ static PROTOCOL_VERSION: &str = "2"; // Include the `submission` module, which is // generated from ./protos/submission.proto. pub mod messages { - #![allow(clippy::derive_partial_eq_without_eq)] + #![allow(clippy::derive_partial_eq_without_eq)] - include!(concat!(env!("OUT_DIR"), "/_.rs")); + include!(concat!(env!("OUT_DIR"), "/_.rs")); } diff --git a/crates/common/src/prov/id/mod.rs b/crates/common/src/prov/id/mod.rs index a5accec27..2a7795b5b 100644 --- a/crates/common/src/prov/id/mod.rs +++ b/crates/common/src/prov/id/mod.rs @@ -739,6 +739,14 @@ impl UuidPart for NamespaceId { } } +impl TryFrom<&'_ str> for NamespaceId { + type Error = ParseIriError; + + fn try_from(value: &str) -> Result { + ProbableChronicleCURIE::from_string(value.to_owned())?.try_into() + } +} + impl TryFrom for NamespaceId { type Error = ParseIriError; diff --git a/crates/common/src/prov/model/json_ld/from_json_ld.rs b/crates/common/src/prov/model/json_ld/from_json_ld.rs index 5e39e45a2..2c8941a4e 100644 --- a/crates/common/src/prov/model/json_ld/from_json_ld.rs +++ b/crates/common/src/prov/model/json_ld/from_json_ld.rs @@ -680,12 +680,8 @@ impl ChronicleOperation { Ok(ChronicleOperation::CreateNamespace(CreateNamespace { id: namespace })) } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::AgentExists)) { let namespace = o.namespace(); - let agent = o.agent(); - let external_id = agent.external_id_part(); - Ok(ChronicleOperation::AgentExists(AgentExists { - namespace, - external_id: external_id.into(), - })) + let agent_id = o.agent(); + Ok(ChronicleOperation::AgentExists(AgentExists { namespace, id: agent_id })) } else if o .has_type(&id_from_iri_string(vocab::ChronicleOperation::AgentActsOnBehalfOf)) { @@ -695,50 +691,66 @@ impl ChronicleOperation { let activity_id = o.optional_activity(); Ok(ChronicleOperation::AgentActsOnBehalfOf(ActsOnBehalfOf::new( - &namespace, - &responsible_id, - &delegate_id, - activity_id.as_ref(), + namespace, + responsible_id, + delegate_id, + activity_id, o.optional_role(), ))) } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::ActivityExists)) { let namespace = o.namespace(); - let activity_id = o.optional_activity().unwrap(); - let external_id = activity_id.external_id_part().to_owned(); - Ok(ChronicleOperation::ActivityExists(ActivityExists { namespace, external_id })) + if let Some(activity_id) = o.optional_activity() { + Ok(ChronicleOperation::ActivityExists(ActivityExists { + namespace, + id: activity_id, + })) + } else { + Err(ProcessorError::MissingActivity) + } } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::StartActivity)) { let namespace = o.namespace(); - let id = o.optional_activity().unwrap(); - let time: DateTime = o.start_time().parse().unwrap(); - Ok(ChronicleOperation::StartActivity(StartActivity { - namespace, - id, - time: time.into(), - })) + if let Some(id) = o.optional_activity() { + match o.start_time().parse::>() { + Ok(time) => Ok(ChronicleOperation::start_activity(namespace, id, time)), + Err(e) => Err(ProcessorError::Time(e)), + } + } else { + Err(ProcessorError::MissingActivity) + } } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::EndActivity)) { let namespace = o.namespace(); - let id = o.optional_activity().unwrap(); - let time: DateTime = o.end_time().parse().unwrap(); - Ok(ChronicleOperation::EndActivity(EndActivity { - namespace, - id, - time: time.into(), - })) + if let Some(id) = o.optional_activity() { + match o.start_time().parse::>() { + Ok(time) => Ok(ChronicleOperation::end_activity(namespace, id, time)), + Err(e) => Err(ProcessorError::Time(e)), + } + } else { + Err(ProcessorError::MissingActivity) + } } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::ActivityUses)) { let namespace = o.namespace(); let id = o.entity(); - let activity = o.optional_activity().unwrap(); - Ok(ChronicleOperation::ActivityUses(ActivityUses { namespace, id, activity })) + if let Some(activity) = o.optional_activity() { + Ok(ChronicleOperation::ActivityUses(ActivityUses { namespace, id, activity })) + } else { + Err(ProcessorError::MissingActivity) + } } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::EntityExists)) { let namespace = o.namespace(); - let entity = o.entity(); - let id = entity.external_id_part().into(); - Ok(ChronicleOperation::EntityExists(EntityExists { namespace, external_id: id })) + let entity_id = o.entity(); + Ok(ChronicleOperation::EntityExists(EntityExists { namespace, id: entity_id })) } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::WasGeneratedBy)) { let namespace = o.namespace(); let id = o.entity(); - let activity = o.optional_activity().unwrap(); - Ok(ChronicleOperation::WasGeneratedBy(WasGeneratedBy { namespace, id, activity })) + if let Some(activity) = o.optional_activity() { + Ok(ChronicleOperation::WasGeneratedBy(WasGeneratedBy { + namespace, + id, + activity, + })) + } else { + Err(ProcessorError::MissingActivity) + } } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::EntityDerive)) { let namespace = o.namespace(); let id = o.entity(); @@ -780,16 +792,16 @@ impl ChronicleOperation { } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::WasAssociatedWith)) { Ok(ChronicleOperation::WasAssociatedWith(WasAssociatedWith::new( - &o.namespace(), - &o.activity(), - &o.agent(), + o.namespace(), + o.activity(), + o.agent(), o.optional_role(), ))) } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::WasAttributedTo)) { Ok(ChronicleOperation::WasAttributedTo(WasAttributedTo::new( - &o.namespace(), - &o.entity(), - &o.agent(), + o.namespace(), + o.entity(), + o.agent(), o.optional_role(), ))) } else if o.has_type(&id_from_iri_string(vocab::ChronicleOperation::WasInformedBy)) { diff --git a/crates/common/src/prov/model/json_ld/mod.rs b/crates/common/src/prov/model/json_ld/mod.rs index d190b5a61..8559d747a 100644 --- a/crates/common/src/prov/model/json_ld/mod.rs +++ b/crates/common/src/prov/model/json_ld/mod.rs @@ -25,6 +25,7 @@ pub enum CompactionError { #[error("Serde conversion: {source}")] Serde { #[from] + #[source] source: serde_json::Error, }, #[error("Expanded document invalid: {message}")] diff --git a/crates/common/src/prov/model/json_ld/to_json_ld.rs b/crates/common/src/prov/model/json_ld/to_json_ld.rs index afab4b7ed..e41cc0293 100644 --- a/crates/common/src/prov/model/json_ld/to_json_ld.rs +++ b/crates/common/src/prov/model/json_ld/to_json_ld.rs @@ -434,7 +434,7 @@ impl ToJson for ChronicleOperation { o }, - ChronicleOperation::AgentExists(AgentExists { namespace, external_id }) => { + ChronicleOperation::AgentExists(AgentExists { namespace, id }) => { let mut o = Value::new_operation(vocab::ChronicleOperation::AgentExists); o.has_value( @@ -448,7 +448,7 @@ impl ToJson for ChronicleOperation { ); o.has_value( - OperationValue::string(external_id), + OperationValue::string(id.external_id_part()), vocab::ChronicleOperation::AgentName, ); @@ -497,7 +497,7 @@ impl ToJson for ChronicleOperation { o }, - ChronicleOperation::ActivityExists(ActivityExists { namespace, external_id }) => { + ChronicleOperation::ActivityExists(ActivityExists { namespace, id }) => { let mut o = Value::new_operation(vocab::ChronicleOperation::ActivityExists); o.has_value( @@ -511,7 +511,7 @@ impl ToJson for ChronicleOperation { ); o.has_value( - OperationValue::string(external_id), + OperationValue::string(id.external_id_part()), vocab::ChronicleOperation::ActivityName, ); @@ -592,7 +592,7 @@ impl ToJson for ChronicleOperation { o }, - ChronicleOperation::EntityExists(EntityExists { namespace, external_id }) => { + ChronicleOperation::EntityExists(EntityExists { namespace, id }) => { let mut o = Value::new_operation(vocab::ChronicleOperation::EntityExists); o.has_value( @@ -606,7 +606,7 @@ impl ToJson for ChronicleOperation { ); o.has_value( - OperationValue::string(external_id), + OperationValue::string(id.external_id_part()), vocab::ChronicleOperation::EntityName, ); diff --git a/crates/common/src/prov/model/mod.rs b/crates/common/src/prov/model/mod.rs index c3080f531..19fb52407 100644 --- a/crates/common/src/prov/model/mod.rs +++ b/crates/common/src/prov/model/mod.rs @@ -57,17 +57,33 @@ pub enum ProcessorError { #[error("Invalid address")] Address, #[error("Json Ld Error {0}")] - Compaction(#[from] json_ld::CompactionError), + Compaction( + #[from] + #[source] + json_ld::CompactionError, + ), #[error("Contradiction {0}")] Contradiction(Contradiction), #[error("Json Ld Error {inner}")] Expansion { inner: String }, #[error("IdentityError {0}")] - Identity(#[from] IdentityError), + Identity( + #[from] + #[source] + IdentityError, + ), #[error("Invalid IRI {0}")] - IRef(#[from] iref::Error), + IRef( + #[from] + #[source] + iref::Error, + ), #[error("Not a Chronicle IRI {0}")] - NotAChronicleIri(#[from] id::ParseIriError), + NotAChronicleIri( + #[from] + #[source] + id::ParseIriError, + ), #[error("Missing @id {object:?}")] MissingId { object: serde_json::Value }, #[error("Missing property {iri}:{object:?}")] @@ -76,18 +92,42 @@ pub enum ProcessorError { NotANode(serde_json::Value), #[error("Chronicle value is not a JSON object")] NotAnObject, + + #[error("Missing activity")] + MissingActivity, #[error("OpaExecutorError: {0}")] - OpaExecutor(#[from] anyhow::Error), + OpaExecutor( + #[from] + #[source] + anyhow::Error, + ), #[error("Malformed JSON {0}")] - SerdeJson(#[from] serde_json::Error), - #[error("Unparsable date/time {0}")] - SubmissionFormat(#[from] PayloadError), + SerdeJson( + #[from] + #[source] + serde_json::Error, + ), + + #[error("Submission {0}")] + SubmissionFormat( + #[from] + #[source] + PayloadError, + ), #[error("Submission body format: {0}")] - Time(#[from] chrono::ParseError), + Time( + #[from] + #[source] + chrono::ParseError, + ), #[error("Tokio Error")] Tokio, #[error("State is not valid utf8 {0}")] - Utf8(#[from] core::str::Utf8Error), + Utf8( + #[from] + #[source] + core::str::Utf8Error, + ), } #[cfg(not(feature = "json-ld"))] @@ -98,9 +138,17 @@ pub enum ProcessorError { #[error("Contradiction {0}")] Contradiction(Contradiction), #[error("IdentityError {0}")] - Identity(#[from] IdentityError), + Identity( + #[from] + #[source] + IdentityError, + ), #[error("Not a Chronicle IRI {0}")] - NotAChronicleIri(#[from] id::ParseIriError), + NotAChronicleIri( + #[from] + #[source] + id::ParseIriError, + ), #[error("Missing @id {object:?}")] MissingId { object: serde_json::Value }, #[error("Missing property {iri}:{object:?}")] @@ -110,17 +158,37 @@ pub enum ProcessorError { #[error("Chronicle value is not a JSON object")] NotAnObject, #[error("OpaExecutorError: {0}")] - OpaExecutor(#[from] anyhow::Error), + OpaExecutor( + #[from] + #[source] + anyhow::Error, + ), #[error("Malformed JSON {0}")] - SerdeJson(#[from] serde_json::Error), + SerdeJson( + #[from] + #[source] + serde_json::Error, + ), #[error("Unparsable date/time {0}")] - SubmissionFormat(#[from] PayloadError), + SubmissionFormat( + #[from] + #[source] + PayloadError, + ), #[error("Submission body format: {0}")] - Time(#[from] chrono::ParseError), + Time( + #[from] + #[source] + chrono::ParseError, + ), #[error("Tokio Error")] Tokio, #[error("State is not valid utf8 {0}")] - Utf8(#[from] core::str::Utf8Error), + Utf8( + #[from] + #[source] + core::str::Utf8Error, + ), } #[derive(Error, Debug)] @@ -238,7 +306,7 @@ impl Agent { namespaceid, external_id, domaintypeid: attributes.get_typ().clone(), - attributes: attributes.get_items().into_iter().cloned().collect(), + attributes: attributes.get_items().iter().cloned().collect(), } } @@ -279,7 +347,7 @@ impl Activity { started, ended, domaintype_id: attributes.get_typ().clone(), - attributes: attributes.get_items().into_iter().cloned().collect(), + attributes: attributes.get_items().iter().cloned().collect(), } } @@ -318,7 +386,7 @@ impl Entity { namespace_id: namespaceid, external_id, domaintypeid: attributes.get_typ().clone(), - attributes: attributes.get_items().into_iter().cloned().collect(), + attributes: attributes.get_items().iter().cloned().collect(), } } @@ -847,8 +915,8 @@ impl ProvModel { self.namespace_context(&id); Ok(()) }, - ChronicleOperation::AgentExists(AgentExists { namespace, external_id, .. }) => { - self.agent_context(&namespace, &AgentId::from_external_id(&external_id)); + ChronicleOperation::AgentExists(AgentExists { namespace, id, .. }) => { + self.agent_context(&namespace, &id); Ok(()) }, @@ -877,10 +945,8 @@ impl ProvModel { Ok(()) }, - ChronicleOperation::ActivityExists(ActivityExists { - namespace, external_id, .. - }) => { - self.activity_context(&namespace, &ActivityId::from_external_id(&external_id)); + ChronicleOperation::ActivityExists(ActivityExists { namespace, id, .. }) => { + self.activity_context(&namespace, &id); Ok(()) }, @@ -988,8 +1054,8 @@ impl ProvModel { Ok(()) }, - ChronicleOperation::EntityExists(EntityExists { namespace, external_id, .. }) => { - self.entity_context(&namespace, &EntityId::from_external_id(&external_id)); + ChronicleOperation::EntityExists(EntityExists { namespace, id, .. }) => { + self.entity_context(&namespace, &id); Ok(()) }, ChronicleOperation::WasGeneratedBy(WasGeneratedBy { namespace, id, activity }) => { @@ -1052,7 +1118,7 @@ impl ProvModel { self.modify_entity(&namespace, &id, move |entity| { entity.domaintypeid = attributes.get_typ().clone(); - entity.attributes = attributes.get_items().into_iter().cloned().collect(); + entity.attributes = attributes.get_items().iter().cloned().collect(); }); Ok(()) @@ -1079,7 +1145,7 @@ impl ProvModel { self.modify_activity(&namespace, &id, move |activity| { activity.domaintype_id = attributes.get_typ().clone(); - activity.attributes = attributes.get_items().into_iter().cloned().collect(); + activity.attributes = attributes.get_items().iter().cloned().collect(); }); Ok(()) @@ -1104,7 +1170,7 @@ impl ProvModel { self.modify_agent(&namespace, &id, move |agent| { agent.domaintypeid = attributes.get_typ().clone(); - agent.attributes = attributes.get_items().into_iter().cloned().collect(); + agent.attributes = attributes.get_items().iter().cloned().collect(); }); Ok(()) diff --git a/crates/common/src/prov/model/proptest.rs b/crates/common/src/prov/model/proptest.rs index 1f8e5bb71..9c0b07467 100644 --- a/crates/common/src/prov/model/proptest.rs +++ b/crates/common/src/prov/model/proptest.rs @@ -74,19 +74,20 @@ prop_compose! { prop_compose! { fn create_agent() (external_id in external_id(),namespace in namespace()) -> AgentExists { - let _id = AgentId::from_external_id(&external_id); + let id = AgentId::from_external_id(&external_id); AgentExists { namespace, - external_id, + id, } } } prop_compose! { fn create_activity() (external_id in external_id(),namespace in namespace()) -> ActivityExists { + let id = ActivityId::from_external_id(&external_id); ActivityExists { namespace, - external_id, + id, } } } @@ -136,9 +137,10 @@ prop_compose! { prop_compose! { fn create_entity() (external_id in external_id(),namespace in namespace()) -> EntityExists { + let id = EntityId::from_external_id(&external_id); EntityExists { namespace, - external_id, + id, } } } @@ -379,11 +381,11 @@ proptest! { prop_assert_eq!(&ns.id, id); }, ChronicleOperation::AgentExists( - AgentExists { namespace, external_id}) => { - let agent = &prov.agents.get(&(namespace.to_owned(),AgentId::from_external_id(external_id))); + AgentExists { namespace, id}) => { + let agent = &prov.agents.get(&(namespace.to_owned(),id.clone())); prop_assert!(agent.is_some()); let agent = agent.unwrap(); - prop_assert_eq!(&agent.external_id, external_id); + prop_assert_eq!(&agent.external_id, id.external_id_part()); prop_assert_eq!(&agent.namespaceid, namespace); }, ChronicleOperation::AgentActsOnBehalfOf( @@ -416,11 +418,11 @@ proptest! { } ChronicleOperation::ActivityExists( - ActivityExists { namespace, external_id }) => { - let activity = &prov.activities.get(&(namespace.clone(),ActivityId::from_external_id(external_id))); + ActivityExists { namespace, id }) => { + let activity = &prov.activities.get(&(namespace.clone(),id.clone())); prop_assert!(activity.is_some()); let activity = activity.unwrap(); - prop_assert_eq!(&activity.external_id, external_id); + prop_assert_eq!(&activity.external_id, id.external_id_part()); prop_assert_eq!(&activity.namespace_id, namespace); }, ChronicleOperation::StartActivity( @@ -492,11 +494,11 @@ proptest! { prop_assert!(has_usage); }, ChronicleOperation::EntityExists( - EntityExists { namespace, external_id}) => { - let entity = &prov.entities.get(&(namespace.to_owned(),EntityId::from_external_id(external_id))); + EntityExists { namespace, id}) => { + let entity = &prov.entities.get(&(namespace.to_owned(),id.clone())); prop_assert!(entity.is_some()); let entity = entity.unwrap(); - prop_assert_eq!(&entity.external_id, external_id); + prop_assert_eq!(&entity.external_id, id.external_id_part()); prop_assert_eq!(&entity.namespace_id, namespace); }, ChronicleOperation::WasGeneratedBy(WasGeneratedBy{namespace, id, activity}) => { diff --git a/crates/common/src/prov/operations.rs b/crates/common/src/prov/operations.rs index 06e08a8c5..3fc161746 100644 --- a/crates/common/src/prov/operations.rs +++ b/crates/common/src/prov/operations.rs @@ -135,12 +135,12 @@ impl CreateNamespace { #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)] pub struct AgentExists { pub namespace: NamespaceId, - pub external_id: ExternalId, + pub id: AgentId, } impl AgentExists { - pub fn new(namespace: NamespaceId, external_id: impl AsRef) -> Self { - Self { namespace, external_id: external_id.as_ref().into() } + pub fn new(namespace: NamespaceId, id: AgentId) -> Self { + Self { namespace, id } } } @@ -165,24 +165,24 @@ pub struct ActsOnBehalfOf { impl ActsOnBehalfOf { pub fn new( - namespace: &NamespaceId, - responsible_id: &AgentId, - delegate_id: &AgentId, - activity_id: Option<&ActivityId>, + namespace: NamespaceId, + responsible_id: AgentId, + delegate_id: AgentId, + activity_id: Option, role: Option, ) -> Self { Self { - namespace: namespace.clone(), + namespace, id: DelegationId::from_component_ids( - delegate_id, - responsible_id, - activity_id, + &delegate_id, + &responsible_id, + activity_id.as_ref(), role.as_ref(), ), role, - activity_id: activity_id.cloned(), - responsible_id: responsible_id.clone(), - delegate_id: delegate_id.clone(), + activity_id, + responsible_id, + delegate_id, } } } @@ -198,7 +198,13 @@ impl ActsOnBehalfOf { #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)] pub struct ActivityExists { pub namespace: NamespaceId, - pub external_id: ExternalId, + pub id: ActivityId, +} + +impl ActivityExists { + pub fn new(namespace: NamespaceId, id: ActivityId) -> Self { + Self { namespace, id } + } } #[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)] @@ -256,10 +262,8 @@ impl parity_scale_codec::Decode for TimeWrapper { ) -> Result { let (timestamp, subsec_nanos) = <(i64, u32)>::decode(input)?; - let datetime = Utc.from_utc_datetime( - &NaiveDateTime::from_timestamp_opt(timestamp, subsec_nanos) - .ok_or("Invalid timestamp")?, - ); + let datetime = + Utc.timestamp_opt(timestamp, subsec_nanos).single().ok_or("Invalid timestamp")?; Ok(Self(datetime)) } @@ -296,6 +300,12 @@ pub struct StartActivity { pub time: TimeWrapper, } +impl StartActivity { + pub fn new(namespace: NamespaceId, id: ActivityId, time: DateTime) -> Self { + Self { namespace, id, time: TimeWrapper(time) } + } +} + #[cfg_attr( feature = "parity-encoding", derive( @@ -312,6 +322,12 @@ pub struct EndActivity { pub time: TimeWrapper, } +impl EndActivity { + pub fn new(namespace: NamespaceId, id: ActivityId, time: DateTime) -> Self { + Self { namespace, id, time: TimeWrapper(time) } + } +} + #[cfg_attr( feature = "parity-encoding", derive( @@ -328,6 +344,19 @@ pub struct ActivityUses { pub activity: ActivityId, } +impl ActivityUses { + /// Creates a new `ActivityUses` instance. + /// + /// # Arguments + /// + /// * `namespace` - The namespace identifier for the activity. + /// * `id` - The unique identifier for the entity being used. + /// * `activity` - The unique identifier for the activity using the entity. + pub fn new(namespace: NamespaceId, id: EntityId, activity: ActivityId) -> Self { + Self { namespace, id, activity } + } +} + #[cfg_attr( feature = "parity-encoding", derive( @@ -340,7 +369,20 @@ pub struct ActivityUses { #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)] pub struct EntityExists { pub namespace: NamespaceId, - pub external_id: ExternalId, + pub id: EntityId, +} + +impl EntityExists { + /// Creates a new `EntityExists` instance. + /// + /// # Arguments + /// + /// * `namespace` - The namespace identifier for the entity. + /// * `id` - The identifier for the entity. + #[tracing::instrument(skip(namespace, id), fields(namespace = %namespace, entity_id = %id))] + pub fn new(namespace: NamespaceId, id: EntityId) -> Self { + Self { namespace, id } + } } #[cfg_attr( @@ -359,6 +401,25 @@ pub struct WasGeneratedBy { pub activity: ActivityId, } +impl WasGeneratedBy { + /// Creates a new `WasGeneratedBy` instance. + /// + /// # Arguments + /// + /// * `namespace` - The namespace identifier for the entity. + /// * `id` - The unique identifier for the entity. + /// * `activity` - The identifier for the activity that generated the entity. + pub fn new(namespace: NamespaceId, id: EntityId, activity: ActivityId) -> Self { + tracing::debug!( + "Creating WasGeneratedBy with namespace: {:?}, id: {:?}, activity: {:?}", + namespace, + id, + activity + ); + Self { namespace, id, activity } + } +} + #[cfg_attr( feature = "parity-encoding", derive( @@ -377,6 +438,27 @@ pub struct EntityDerive { pub typ: DerivationType, } +impl EntityDerive { + /// Creates a new `EntityDerive` instance. + /// + /// # Arguments + /// + /// * `namespace` - The namespace identifier for the entity. + /// * `id` - The unique identifier for the entity. + /// * `used_id` - The identifier for the entity that was used. + /// * `activity_id` - The identifier for the activity that derived the entity, if any. + /// * `typ` - The type of derivation. + pub fn new( + namespace: NamespaceId, + id: EntityId, + used_id: EntityId, + activity_id: Option, + typ: DerivationType, + ) -> Self { + Self { namespace, id, used_id, activity_id, typ } + } +} + #[cfg_attr( feature = "parity-encoding", derive( @@ -397,17 +479,17 @@ pub struct WasAssociatedWith { impl WasAssociatedWith { pub fn new( - namespace: &NamespaceId, - activity_id: &ActivityId, - agent_id: &AgentId, + namespace: NamespaceId, + activity_id: ActivityId, + agent_id: AgentId, role: Option, ) -> Self { Self { - id: AssociationId::from_component_ids(agent_id, activity_id, role.as_ref()), + id: AssociationId::from_component_ids(&agent_id, &activity_id, role.as_ref()), role, - namespace: namespace.clone(), - activity_id: activity_id.clone(), - agent_id: agent_id.clone(), + namespace, + activity_id, + agent_id, } } } @@ -431,18 +513,19 @@ pub struct WasAttributedTo { } impl WasAttributedTo { + #[tracing::instrument(skip(namespace, role))] pub fn new( - namespace: &NamespaceId, - entity_id: &EntityId, - agent_id: &AgentId, + namespace: NamespaceId, + entity_id: EntityId, + agent_id: AgentId, role: Option, ) -> Self { Self { - id: AttributionId::from_component_ids(agent_id, entity_id, role.as_ref()), + id: AttributionId::from_component_ids(&agent_id, &entity_id, role.as_ref()), role, - namespace: namespace.clone(), - entity_id: entity_id.clone(), - agent_id: agent_id.clone(), + namespace, + entity_id, + agent_id, } } } @@ -463,6 +546,23 @@ pub struct WasInformedBy { pub informing_activity: ActivityId, } +impl WasInformedBy { + /// Creates a new `WasInformedBy` instance. + /// + /// # Arguments + /// + /// * `namespace` - The namespace identifier for the activity. + /// * `activity` - The ActivityId for the activity that was informed. + /// * `informing_activity` - The ActivityId for the informing activity. + pub fn new( + namespace: NamespaceId, + activity: ActivityId, + informing_activity: ActivityId, + ) -> Self { + Self { namespace, activity, informing_activity } + } +} + #[cfg_attr( feature = "parity-encoding", derive( @@ -521,6 +621,131 @@ pub enum ChronicleOperation { } impl ChronicleOperation { + #[tracing::instrument] + #[tracing::instrument] + pub fn create_namespace(id: NamespaceId) -> Self { + ChronicleOperation::CreateNamespace(CreateNamespace::new(id)) + } + + #[tracing::instrument] + pub fn agent_exists(namespace: NamespaceId, id: AgentId) -> Self { + ChronicleOperation::AgentExists(AgentExists::new(namespace, id)) + } + + #[tracing::instrument] + pub fn agent_acts_on_behalf_of( + namespace: NamespaceId, + responsible_id: AgentId, + delegate_id: AgentId, + activity_id: Option, + role: Option, + ) -> Self { + ChronicleOperation::AgentActsOnBehalfOf(ActsOnBehalfOf::new( + namespace, + responsible_id, + delegate_id, + activity_id, + role, + )) + } + + #[tracing::instrument] + pub fn activity_exists(namespace: NamespaceId, id: ActivityId) -> Self { + ChronicleOperation::ActivityExists(ActivityExists::new(namespace, id)) + } + + #[tracing::instrument] + pub fn start_activity( + namespace: NamespaceId, + id: ActivityId, + start_time: DateTime, + ) -> Self { + ChronicleOperation::StartActivity(StartActivity::new(namespace, id, start_time)) + } + + #[tracing::instrument] + pub fn end_activity(namespace: NamespaceId, id: ActivityId, end_time: DateTime) -> Self { + ChronicleOperation::EndActivity(EndActivity::new(namespace, id, end_time)) + } + + #[tracing::instrument] + pub fn activity_uses( + namespace: NamespaceId, + activity_id: ActivityId, + entity_id: EntityId, + ) -> Self { + ChronicleOperation::ActivityUses(ActivityUses::new(namespace, entity_id, activity_id)) + } + + #[tracing::instrument] + pub fn entity_exists(namespace: NamespaceId, id: EntityId) -> Self { + ChronicleOperation::EntityExists(EntityExists::new(namespace, id)) + } + + #[tracing::instrument] + pub fn was_generated_by( + namespace: NamespaceId, + entity_id: EntityId, + activity_id: ActivityId, + ) -> Self { + ChronicleOperation::WasGeneratedBy(WasGeneratedBy::new(namespace, entity_id, activity_id)) + } + + pub fn entity_derive( + namespace: NamespaceId, + source_id: EntityId, + target_id: EntityId, + activity_id: Option, + derivation_type: DerivationType, + ) -> Self { + ChronicleOperation::EntityDerive(EntityDerive::new( + namespace, + source_id, + target_id, + activity_id, + derivation_type, + )) + } + + pub fn set_attributes(set_attributes: SetAttributes) -> Self { + ChronicleOperation::SetAttributes(set_attributes) + } + + #[tracing::instrument] + pub fn was_associated_with( + namespace: NamespaceId, + activity_id: ActivityId, + agent_id: AgentId, + role: Option, + ) -> Self { + ChronicleOperation::WasAssociatedWith(WasAssociatedWith::new( + namespace, + activity_id, + agent_id, + role, + )) + } + + pub fn was_attributed_to( + namespace: NamespaceId, + entity_id: EntityId, + agent_id: AgentId, + role: Option, + ) -> Self { + ChronicleOperation::WasAttributedTo(WasAttributedTo::new( + namespace, entity_id, agent_id, role, + )) + } + + #[tracing::instrument] + pub fn was_informed_by( + namespace: NamespaceId, + informed: ActivityId, + informant: ActivityId, + ) -> Self { + ChronicleOperation::WasInformedBy(WasInformedBy::new(namespace, informed, informant)) + } + /// Returns a reference to the `NamespaceId` of the `ChronicleOperation` pub fn namespace(&self) -> &NamespaceId { match self { diff --git a/crates/common/src/prov/vocab.rs b/crates/common/src/prov/vocab.rs index 9f2444f84..8bd8a4b53 100644 --- a/crates/common/src/prov/vocab.rs +++ b/crates/common/src/prov/vocab.rs @@ -262,7 +262,7 @@ mod chronicle { use iri_string::types::UriString; #[cfg(not(feature = "std"))] use parity_scale_codec::alloc::string::String; - use percent_encoding::NON_ALPHANUMERIC; + #[cfg(not(feature = "std"))] use scale_info::prelude::{borrow::ToOwned, string::ToString, *}; use uuid::Uuid; @@ -326,7 +326,8 @@ mod chronicle { pub const LONG_PREFIX: &'static str = "http://chronicle.works/chronicle/ns#"; pub const PREFIX: &'static str = "chronicle"; - /// Encodes the given external ID using percent-encoding to ensure it is a valid Chronicle CURIE + /// Encodes the given external ID using percent-encoding to ensure it is a valid Chronicle + /// CURIE fn encode_external_id(external_id: &ExternalId) -> String { percent_encoding::utf8_percent_encode(external_id.as_str(), &ENCODE_SET).to_string() } @@ -394,7 +395,8 @@ mod chronicle { .try_into() } - /// Constructs a delegation IRI using given delegate and responsible agent IDs, and optional activity and role. + /// Constructs a delegation IRI using given delegate and responsible agent IDs, and optional + /// activity and role. #[tracing::instrument( name = "delegation_iri_creation", skip(delegate, responsible, activity, role) diff --git a/crates/opactl/src/main.rs b/crates/opactl/src/main.rs index 8e536145c..24483e64e 100644 --- a/crates/opactl/src/main.rs +++ b/crates/opactl/src/main.rs @@ -44,13 +44,25 @@ pub enum OpaCtlError { Cancelled(oneshot::Canceled), #[error("Communication error: {0}")] - Communication(#[from] SubxtClientError), + Communication( + #[from] + #[source] + SubxtClientError, + ), #[error("IO error: {0}")] - IO(#[from] std::io::Error), + IO( + #[from] + #[source] + std::io::Error, + ), #[error("Json error: {0}")] - Json(#[from] serde_json::Error), + Json( + #[from] + #[source] + serde_json::Error, + ), #[error("Pkcs8 error")] Pkcs8, @@ -62,13 +74,25 @@ pub enum OpaCtlError { TransactionNotFound(ChronicleTransactionId), #[error("Error loading from URL: {0}")] - Url(#[from] FromUrlError), + Url( + #[from] + #[source] + FromUrlError, + ), #[error("Utf8 error: {0}")] - Utf8(#[from] std::str::Utf8Error), + Utf8( + #[from] + #[source] + std::str::Utf8Error, + ), #[error("Signing: {0}")] - Signing(#[from] SecretError), + Signing( + #[from] + #[source] + SecretError, + ), #[error("Missing Argument")] MissingArgument(String), @@ -77,7 +101,11 @@ pub enum OpaCtlError { NotFound, #[error("Could not build transaction {0}")] - InvalidTransaction(#[from] TransactionError), + InvalidTransaction( + #[from] + #[source] + TransactionError, + ), } impl From for OpaCtlError { diff --git a/crates/opactl/src/test/mod.rs b/crates/opactl/src/test/mod.rs index a8b4d8f0a..8e4b129f1 100644 --- a/crates/opactl/src/test/mod.rs +++ b/crates/opactl/src/test/mod.rs @@ -11,8 +11,6 @@ use k256::{ use rand::rngs::StdRng; use rand_core::SeedableRng; -use serde_json::{self, Value}; - use std::io::Write; use tempfile::{NamedTempFile, TempDir}; @@ -129,16 +127,15 @@ async fn rotate_root() { ".**.key" => "[pem]", ".**.correlation_id" => "[correlation_id]" }, @r###" - --- - - - 7ed19313e8ece6c4f5551b9bd1090797ad25c6d85f7b523b2214d4fe448372279aa95c - - current: - key: "[pem]" - version: 1 - expired: - key: "[pem]" - version: 0 - id: root - "###); + --- + - id: root + current: + key: "[pem]" + version: 1 + expired: + key: "[pem]" + version: 0 + "###); } #[tokio::test] @@ -183,20 +180,18 @@ async fn register_and_rotate_key() { ".**.key" => "[pem]", ".**.correlation_id" => "[correlation_id]" }, @r###" - --- - - - 7ed19313e8ece6c4f5551b9bd1090797ad25c6d85f7b523b2214d4fe448372279aa95c - - current: - key: "[pem]" - version: 0 - expired: ~ - id: root - - - 7ed19336d8b5677c39a7b872910f948944dd84ba014846c81fcd53fe1fd5289b9dfd1c - - current: - key: "[pem]" - version: 0 - expired: ~ - id: test - "###); + --- + - id: test + current: + key: "[pem]" + version: 0 + expired: ~ + - id: root + current: + key: "[pem]" + version: 0 + expired: ~ + "###); let new_key_2 = key_from_seed(1); @@ -285,7 +280,8 @@ async fn set_and_update_policy() { --- WaitedAndFound: PolicyUpdate: - - id: test + policy: + id: test hash: - 112 - 37 @@ -320,22 +316,7 @@ async fn set_and_update_policy() { - 122 - 58 - 230 - - - 126 - - 73 - - 89 - - 163 - - 235 - - 197 - - 73 - - 130 - - 154 - - 213 - - 245 - - 47 - - 249 - - 40 - - 118 - - 225 + correlation_id: "[correlation_id]" "###); insta::assert_yaml_snapshot!(opa_tp.stored_policy(), { diff --git a/crates/opactl/src/test/stubstrate.rs b/crates/opactl/src/test/stubstrate.rs index 4f76aabc1..eeb7dfaf6 100644 --- a/crates/opactl/src/test/stubstrate.rs +++ b/crates/opactl/src/test/stubstrate.rs @@ -156,37 +156,6 @@ impl SubstrateStateReader for Stubstrate { pallet_name, entry_name ); - let result = self.rt.lock().unwrap().execute_with(|| { - let key = address.encode(); - match pallet_name { - "OpaModule" => match entry_name { - "PolicyStore" => { - if let Some(value) = pallet_opa::PolicyStore::::get(key) { - tracing::info!("Successfully retrieved policy state entry"); - return Ok(Some(scale_value::serde::to_value(value))); - } else { - tracing::warn!("No policy state entry found for the given key"); - return Ok(None); - } - }, - "KeyStore" => { - if let Some(value) = pallet_opa::KeyStore::::get(key) { - tracing::info!("Successfully retrieved key state entry"); - return Ok(Some(scale_value::serde::to_value(value))); - } else { - tracing::warn!("No key state entry found for the given key"); - return Ok(None); - } - }, - _ => { - panic!("Unknown entry name: {}", entry_name); - }, - }, - _ => { - panic!("Unknown pallet name: {}", pallet_name); - }, - } - }); - result + unimplemented!() } } diff --git a/crates/protocol-abstract/src/abstract_ledger.rs b/crates/protocol-abstract/src/abstract_ledger.rs index 2e07aea53..9fb9428a9 100644 --- a/crates/protocol-abstract/src/abstract_ledger.rs +++ b/crates/protocol-abstract/src/abstract_ledger.rs @@ -12,7 +12,11 @@ use tracing::{instrument, warn}; #[derive(Debug, Error)] pub enum BlockIdError { #[error("Parse {0}")] - Parse(#[from] anyhow::Error), + Parse( + #[from] + #[source] + anyhow::Error, + ), } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BlockId { diff --git a/crates/protocol-substrate-opa/src/lib.rs b/crates/protocol-substrate-opa/src/lib.rs index 8551b3348..f6d3e4511 100644 --- a/crates/protocol-substrate-opa/src/lib.rs +++ b/crates/protocol-substrate-opa/src/lib.rs @@ -59,18 +59,16 @@ where type KeyUpdate = (common::opa::codec::KeysV1, ChronicleTransactionId); match (event.pallet_name(), event.variant_name(), event.field_bytes()) { ("Opa", "PolicyUpdate", mut event_bytes) => match PolicyUpdate::decode(&mut event_bytes) { - Ok((meta, correlation_id)) => { - Ok(Some(OpaEvent::new_policy_update(meta.try_into()?, correlation_id))) - }, + Ok((meta, correlation_id)) => + Ok(Some(OpaEvent::new_policy_update(meta.try_into()?, correlation_id))), Err(e) => { tracing::error!("Failed to decode ProvModel: {}", e); Err(e.into()) }, }, ("Chronicle", "KeyUpdate", mut event_bytes) => match KeyUpdate::decode(&mut event_bytes) { - Ok((keys, correlation_id)) => { - Ok(OpaEvent::new_key_update(keys.try_into()?, correlation_id).into()) - }, + Ok((keys, correlation_id)) => + Ok(OpaEvent::new_key_update(keys.try_into()?, correlation_id).into()), Err(e) => { tracing::error!("Failed to decode Contradiction: {}", e); Err(e.into()) diff --git a/crates/protocol-substrate-opa/src/transaction.rs b/crates/protocol-substrate-opa/src/transaction.rs index 98c3c2412..f32887a01 100644 --- a/crates/protocol-substrate-opa/src/transaction.rs +++ b/crates/protocol-substrate-opa/src/transaction.rs @@ -10,9 +10,17 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum TransactionError { #[error("Secret error: {0}")] - SecretError(#[from] SecretError), + SecretError( + #[from] + #[source] + SecretError, + ), #[error("Secret string error: {0}")] - SecretStringError(#[from] SecretStringError), + SecretStringError( + #[from] + #[source] + SecretStringError, + ), } #[derive(Clone)] diff --git a/crates/protocol-substrate/src/subxt_client.rs b/crates/protocol-substrate/src/subxt_client.rs index 2152dd55c..e29ff2388 100644 --- a/crates/protocol-substrate/src/subxt_client.rs +++ b/crates/protocol-substrate/src/subxt_client.rs @@ -44,21 +44,14 @@ type ExtrinsicResult = impl SubstrateClient where - // Constraint for the client configuration C: subxt::Config< - Hash = subxt::utils::H256, /* Specifies the hash type used by the blockchain - polkadot - * default */ - Address = MultiAddress, /* Specifies the address format as polkadot - * defaults */ - Signature = MultiSignature, // Specifies the signature format - polkadot default + Hash = subxt::utils::H256, + Address = MultiAddress, + Signature = MultiSignature, >, - // Ensures that the extrinsic parameters can be defaulted >::OtherParams: Default, - // LedgerTransaction trait bound with Send and Sync for multi-threaded usage T: LedgerTransaction + Send + Sync, - // Ensures that the payload of the transaction can be encoded as fields ::Payload: subxt::ext::scale_encode::EncodeAsFields, - // LedgerEventCodec trait bound with Send and Sync for multi-threaded usage EC: LedgerEventCodec + Send + Sync, { pub async fn connect(url: impl AsRef) -> Result { @@ -130,19 +123,35 @@ where #[derive(Debug, thiserror::Error)] pub enum SubxtClientError { #[error("Subxt error: {0}")] - SubxtError(#[from] subxt::Error), + SubxtError( + #[from] + #[source] + subxt::Error, + ), #[error("Invalid block")] InvalidBlock, #[error("Codec: {0}")] - Codec(#[from] subxt::ext::codec::Error), + Codec( + #[from] + #[source] + subxt::ext::codec::Error, + ), #[error("Decode: {0}")] - Decode(#[from] subxt::error::DecodeError), + Decode( + #[from] + #[source] + subxt::error::DecodeError, + ), #[error("Serde: {0}")] - Serde(#[from] subxt::ext::scale_value::serde::SerializerError), + Serde( + #[from] + #[source] + subxt::ext::scale_value::serde::SerializerError, + ), } impl From for SubxtClientError {