diff --git a/Cargo.lock b/Cargo.lock index 5fac82c4..71419f8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,7 @@ dependencies = [ "custom_error", "derivative", "diesel", + "diesel-async", "diesel_migrations", "futures", "hex", @@ -262,7 +263,7 @@ dependencies = [ "tracing", "url", "user-error", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -299,7 +300,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1007,7 +1008,7 @@ dependencies = [ "static_assertions_next", "tempfile", "thiserror", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -1018,12 +1019,12 @@ checksum = "fc51fd6b7102acda72bc94e8ae1543844d5688ff394a6cf7c21f2a07fe2d64e4" dependencies = [ "Inflector", "async-graphql-parser", - "darling 0.20.9", + "darling 0.20.10", "proc-macro-crate 3.1.0", "proc-macro2", "quote", "strum 0.26.3", - "syn 2.0.68", + "syn 2.0.70", "thiserror", ] @@ -1177,7 +1178,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1188,13 +1189,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1258,7 +1259,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "itoa", "matchit", "memchr", @@ -1412,7 +1413,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1680,7 +1681,7 @@ checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1803,9 +1804,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.104" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" dependencies = [ "jobserver", "libc", @@ -1910,6 +1911,8 @@ dependencies = [ "common 0.7.5", "const_format", "diesel", + "diesel-async", + "diesel_migrations", "dotenvy", "embedded-substrate", "futures", @@ -1942,7 +1945,7 @@ dependencies = [ "tracing-log 0.1.4", "url", "user-error", - "uuid 1.9.1", + "uuid 1.10.0", "valico", ] @@ -1965,6 +1968,7 @@ dependencies = [ "chrono", "common 0.7.5", "diesel", + "diesel-async", "futures", "insta", "lazy_static", @@ -1978,7 +1982,7 @@ dependencies = [ "tokio", "tonic 0.10.2", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -1990,7 +1994,7 @@ dependencies = [ "arrow-flight", "arrow-ipc", "arrow-schema", - "clap 4.5.8", + "clap 4.5.9", "prettytable-rs", "tokio", "tonic 0.10.2", @@ -2032,7 +2036,7 @@ dependencies = [ "tokio", "tracing", "tracing-log 0.1.4", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -2045,14 +2049,16 @@ dependencies = [ "common 0.7.5", "derivative", "diesel", + "diesel-async", "diesel_migrations", + "futures", "hex", "protocol-substrate-chronicle", "r2d2", "serde_json", "thiserror", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -2124,6 +2130,8 @@ dependencies = [ "chrono", "common 0.7.5", "diesel", + "diesel-async", + "diesel_migrations", "frame-support", "frame-system", "futures", @@ -2147,7 +2155,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -2292,9 +2300,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.8" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" +checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" dependencies = [ "clap_builder", "clap_derive 4.5.8", @@ -2302,9 +2310,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.8" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" +checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" dependencies = [ "anstream", "anstyle", @@ -2344,7 +2352,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2424,7 +2432,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/w3f/ring-proof#96137b150288a66bc9e4df495efc64769b5d1321" +source = "git+https://github.com/w3f/ring-proof#626c9598be949aa3dbdd72e8a40531f68b01d6c2" dependencies = [ "ark-ec", "ark-ff", @@ -2488,7 +2496,7 @@ dependencies = [ "tokio", "tracing", "url", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -2791,7 +2799,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.8", + "clap 4.5.9", "criterion-plot", "futures", "is-terminal", @@ -2991,7 +2999,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3024,7 +3032,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3041,7 +3049,7 @@ checksum = "4b2c1c1776b986979be68bb2285da855f8d8a35851a769fca8740df7c3d07877" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3066,12 +3074,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ - "darling_core 0.20.9", - "darling_macro 0.20.9", + "darling_core 0.20.10", + "darling_macro 0.20.10", ] [[package]] @@ -3104,16 +3112,16 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3140,13 +3148,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ - "darling_core 0.20.9", + "darling_core 0.20.10", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3188,6 +3196,25 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "deadpool" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e" +dependencies = [ + "async-trait", + "deadpool-runtime", + "num_cpus", + "retain_mut", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + [[package]] name = "decoded-char" version = "0.1.1" @@ -3268,7 +3295,7 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3279,7 +3306,7 @@ checksum = "62d671cc41a825ebabc75757b62d3d168c577f9149b2d49ece1dad1f72119d25" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3323,14 +3350,14 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "diesel" -version = "2.2.1" +version = "2.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62d6dcd069e7b5fe49a302411f759d4cf1cf2c27fe798ef46fb8baefc053dd2b" +checksum = "ff236accb9a5069572099f0b350a92e9560e8e63a9b8d546162f4a5e03026bb2" dependencies = [ "bitflags 2.6.0", "byteorder", @@ -3339,27 +3366,41 @@ dependencies = [ "itoa", "pq-sys", "r2d2", - "uuid 1.9.1", + "uuid 1.10.0", +] + +[[package]] +name = "diesel-async" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acada1517534c92d3f382217b485db8a8638f111b0e3f2a2a8e26165050f77be" +dependencies = [ + "async-trait", + "deadpool", + "diesel", + "futures-util", + "scoped-futures", + "tokio", + "tokio-postgres", ] [[package]] name = "diesel_derives" -version = "2.2.1" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59de76a222c2b8059f789cbe07afbfd8deb8c31dd0bc2a21f85e256c1def8259" +checksum = "14701062d6bed917b5c7103bdffaee1e4609279e240488ad24e7bd979ca6866c" dependencies = [ "diesel_table_macro_syntax", - "dsl_auto_type", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "diesel_migrations" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a73ce704bad4231f001bff3314d91dce4aba0770cee8b233991859abc15c1f6" +checksum = "6036b3f0120c5961381b570ee20a02432d7e2d27ea60de9578799cf9156914ac" dependencies = [ "diesel", "migrations_internals", @@ -3368,11 +3409,11 @@ dependencies = [ [[package]] name = "diesel_table_macro_syntax" -version = "0.2.0" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" +checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3480,7 +3521,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3526,7 +3567,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.68", + "syn 2.0.70", "termcolor", "toml 0.8.14", "walkdir", @@ -3550,20 +3591,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" -[[package]] -name = "dsl_auto_type" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0892a17df262a24294c382f0d5997571006e7a4348b4327557c4ff1cd4a8bccc" -dependencies = [ - "darling 0.20.9", - "either", - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.68", -] - [[package]] name = "dtoa" version = "1.0.9" @@ -3747,7 +3774,7 @@ dependencies = [ "thiserror", "tokio", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -3876,7 +3903,7 @@ dependencies = [ "prettyplease 0.2.20", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4158,7 +4185,7 @@ dependencies = [ "Inflector", "array-bytes 6.2.3", "chrono", - "clap 4.5.8", + "clap 4.5.9", "comfy-table", "frame-benchmarking", "frame-support", @@ -4319,7 +4346,7 @@ dependencies = [ "proc-macro2", "quote", "sp-crypto-hashing 0.0.0", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4331,7 +4358,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4341,7 +4368,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.9.0 dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4495,7 +4522,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -5116,9 +5143,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -5140,9 +5167,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", @@ -5167,7 +5194,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "log", "rustls 0.21.12", "rustls-native-certs 0.6.3", @@ -5183,10 +5210,10 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "log", - "rustls 0.23.10", + "rustls 0.23.11", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -5199,7 +5226,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.29", + "hyper 0.14.30", "pin-project-lite 0.2.14", "tokio", "tokio-io-timeout", @@ -5212,7 +5239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.29", + "hyper 0.14.30", "native-tls", "tokio", "tokio-native-tls", @@ -5226,7 +5253,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "native-tls", "tokio", @@ -5245,7 +5272,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.4.0", + "hyper 1.4.1", "pin-project-lite 0.2.14", "socket2 0.5.7", "tokio", @@ -5856,7 +5883,7 @@ dependencies = [ "http 1.1.0", "jsonrpsee-core 0.23.2", "pin-project", - "rustls 0.23.10", + "rustls 0.23.11", "rustls-pki-types", "rustls-platform-verifier", "soketto 0.8.0", @@ -5879,7 +5906,7 @@ dependencies = [ "beef", "futures-timer", "futures-util", - "hyper 0.14.29", + "hyper 0.14.30", "jsonrpsee-types 0.22.5", "parking_lot 0.12.3", "pin-project", @@ -5926,7 +5953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ccf93fc4a0bfe05d851d37d7c32b7f370fe94336b52a2f0efc5f1981895c2e5" dependencies = [ "async-trait", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-rustls 0.24.2", "jsonrpsee-core 0.22.5", "jsonrpsee-types 0.22.5", @@ -5948,12 +5975,12 @@ dependencies = [ "async-trait", "base64 0.22.1", "http-body 1.0.0", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-rustls 0.27.2", "hyper-util", "jsonrpsee-core 0.23.2", "jsonrpsee-types 0.23.2", - "rustls 0.23.10", + "rustls 0.23.11", "rustls-platform-verifier", "serde", "serde_json", @@ -5974,7 +6001,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -5985,7 +6012,7 @@ checksum = "12d8b6a9674422a8572e0b0abb12feeb3f2aeda86528c80d0350c2bd0923ab41" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "jsonrpsee-core 0.22.5", "jsonrpsee-types 0.22.5", "pin-project", @@ -6063,7 +6090,7 @@ dependencies = [ "anyhow", "base64 0.21.7", "bytecount", - "clap 4.5.8", + "clap 4.5.9", "fancy-regex", "fraction", "getrandom 0.2.15", @@ -6080,7 +6107,7 @@ dependencies = [ "serde_json", "time", "url", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -6955,7 +6982,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -6969,7 +6996,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -6980,7 +7007,7 @@ checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -6991,7 +7018,7 @@ checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -7046,6 +7073,16 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "memchr" version = "2.7.4" @@ -7136,7 +7173,7 @@ checksum = "5d58e362dc7206e9456ddbcdbd53c71ba441020e62104703075a69151e38d85f" dependencies = [ "base64 0.22.1", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-tls 0.6.0", "hyper-util", "indexmap 2.2.6", @@ -7166,19 +7203,19 @@ dependencies = [ [[package]] name = "migrations_internals" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd01039851e82f8799046eabbb354056283fb265c8ec0996af940f4e85a380ff" +checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada" dependencies = [ "serde", - "toml 0.8.14", + "toml 0.7.8", ] [[package]] name = "migrations_macros" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb161cc72176cb37aa47f1fc520d3ef02263d67d661f44f05d05a079e1237fd" +checksum = "cce3325ac70e67bbab5bd837a31cae01f1a6db64e0e744a33cb03a543469ef08" dependencies = [ "migrations_internals", "proc-macro2", @@ -7278,7 +7315,7 @@ dependencies = [ "assert-json-diff", "colored", "futures-core", - "hyper 0.14.29", + "hyper 0.14.30", "log", "rand 0.8.5", "regex", @@ -7416,7 +7453,7 @@ checksum = "254a5372af8fc138e36684761d3c0cdb758a4410e938babcff1c860ce14ddbfc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -7581,7 +7618,7 @@ checksum = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65" name = "node-chronicle" version = "0.1.0" dependencies = [ - "clap 4.5.8", + "clap 4.5.9", "common 0.7.5", "frame-benchmarking", "frame-benchmarking-cli", @@ -7859,9 +7896,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" -version = "11.1.3" +version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "opa" @@ -7889,7 +7926,7 @@ dependencies = [ "async-trait", "chronicle-signing", "chronicle-telemetry", - "clap 4.5.8", + "clap 4.5.9", "common 0.7.5", "const_format", "frame-support", @@ -7923,7 +7960,7 @@ dependencies = [ "tracing", "url", "user-error", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -7961,7 +7998,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -8211,7 +8248,7 @@ dependencies = [ "sp-runtime 31.0.1", "sp-std 14.0.0 (git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.9.0)", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -8278,7 +8315,7 @@ dependencies = [ "sp-runtime 31.0.1", "sp-std 14.0.0 (git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.9.0)", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -8584,7 +8621,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -8701,7 +8738,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -8799,7 +8836,7 @@ dependencies = [ "headers", "http 1.1.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "mime", "nix 0.28.0", @@ -8834,7 +8871,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -8901,7 +8938,7 @@ dependencies = [ "polkavm-common 0.8.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -8913,7 +8950,7 @@ dependencies = [ "polkavm-common 0.9.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -8923,7 +8960,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15e85319a0d5129dc9f021c62607e0804f5fb777a05cdda44d750ac0732def66" dependencies = [ "polkavm-derive-impl 0.8.0", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -8933,7 +8970,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ba81f7b5faac81e528eb6158a6f3c9e0bb1008e0ffa19653bc8dea925ecb429" dependencies = [ "polkavm-derive-impl 0.9.0", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9010,6 +9047,35 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "postgres-protocol" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" +dependencies = [ + "base64 0.21.7", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac 0.12.1", + "md-5", + "memchr", + "rand 0.8.5", + "sha2 0.10.8", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "postgres-protocol", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -9024,9 +9090,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pq-sys" -version = "0.6.1" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24ff9e4cf6945c988f0db7005d87747bf72864965c3529d259ad155ac41d584" +checksum = "31c0052426df997c0cbd30789eb44ca097e3541717a7b8fa36b1c464ee7edebd" dependencies = [ "vcpkg", ] @@ -9098,7 +9164,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9179,7 +9245,7 @@ checksum = "834da187cfe638ae8abb0203f0b33e5ccdb02a28e7199f2f47b3e2754f50edca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9225,7 +9291,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9313,7 +9379,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9361,6 +9427,7 @@ dependencies = [ "derivative", "futures", "hex", + "jsonrpsee-core 0.23.2", "k256 0.11.6", "pallet-chronicle", "protocol-abstract", @@ -9371,7 +9438,7 @@ dependencies = [ "thiserror", "tokio", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -9398,7 +9465,7 @@ dependencies = [ "thiserror", "tokio", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -9425,7 +9492,7 @@ dependencies = [ "thiserror", "tokio", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] @@ -9797,7 +9864,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9889,7 +9956,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-rustls 0.24.2", "hyper-tls 0.5.0", "ipnet", @@ -9929,6 +9996,12 @@ dependencies = [ "quick-error", ] +[[package]] +name = "retain_mut" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" + [[package]] name = "rfc6979" version = "0.3.1" @@ -9962,13 +10035,14 @@ dependencies = [ [[package]] name = "ring" version = "0.1.0" -source = "git+https://github.com/w3f/ring-proof#96137b150288a66bc9e4df495efc64769b5d1321" +source = "git+https://github.com/w3f/ring-proof#626c9598be949aa3dbdd72e8a40531f68b01d6c2" dependencies = [ "ark-ec", "ark-ff", "ark-poly", "ark-serialize", "ark-std", + "arrayvec 0.7.4", "blake2 0.10.6", "common 0.1.0", "fflonk", @@ -10126,7 +10200,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.68", + "syn 2.0.70", "walkdir", ] @@ -10278,9 +10352,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.10" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" dependencies = [ "log", "once_cell", @@ -10352,7 +10426,7 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.10", + "rustls 0.23.11", "rustls-native-certs 0.7.1", "rustls-platform-verifier-android", "rustls-webpki 0.102.5", @@ -10567,7 +10641,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -10577,7 +10651,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.9.0 dependencies = [ "array-bytes 6.2.3", "chrono", - "clap 4.5.8", + "clap 4.5.9", "fdlimit", "futures", "itertools 0.10.5", @@ -11094,7 +11168,7 @@ dependencies = [ "fnv", "futures", "futures-timer", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-rustls 0.24.2", "libp2p", "log", @@ -11187,7 +11261,7 @@ dependencies = [ "futures", "governor", "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "jsonrpsee 0.22.5", "log", "serde_json", @@ -11400,7 +11474,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -11621,7 +11695,7 @@ dependencies = [ "proc-macro2", "quote", "scale-info", - "syn 2.0.68", + "syn 2.0.70", "thiserror", ] @@ -11715,6 +11789,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "scoped-futures" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1473e24c637950c9bd38763220bea91ec3e095a89f672bbd7a10d03e77ba467" +dependencies = [ + "cfg-if", + "pin-utils", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -11872,9 +11956,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] @@ -11906,13 +11990,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -12412,7 +12496,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -12696,7 +12780,7 @@ dependencies = [ [[package]] name = "sp-crypto-ec-utils" version = "0.10.0" -source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" +source = "git+https://github.com/paritytech/polkadot-sdk#6dd777ffe62cff936e04a76134ccf07de9dee429" dependencies = [ "ark-bls12-377", "ark-bls12-377-ext", @@ -12747,7 +12831,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.9.0 dependencies = [ "quote", "sp-crypto-hashing 0.0.0", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -12767,7 +12851,7 @@ checksum = "48d09fa0a5f7299fb81ee25ae3853d26200f7a348148aed6de76be905c007dbe" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -12777,17 +12861,17 @@ source = "git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.9.0 dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "sp-debug-derive" version = "14.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" +source = "git+https://github.com/paritytech/polkadot-sdk#6dd777ffe62cff936e04a76134ccf07de9dee429" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -12804,7 +12888,7 @@ dependencies = [ [[package]] name = "sp-externalities" version = "0.25.0" -source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" +source = "git+https://github.com/paritytech/polkadot-sdk#6dd777ffe62cff936e04a76134ccf07de9dee429" dependencies = [ "environmental", "parity-scale-codec", @@ -13078,7 +13162,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" version = "24.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" +source = "git+https://github.com/paritytech/polkadot-sdk#6dd777ffe62cff936e04a76134ccf07de9dee429" dependencies = [ "bytes", "impl-trait-for-tuples", @@ -13124,20 +13208,20 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "sp-runtime-interface-proc-macro" version = "17.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" +source = "git+https://github.com/paritytech/polkadot-sdk#6dd777ffe62cff936e04a76134ccf07de9dee429" dependencies = [ "Inflector", "expander", "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -13151,7 +13235,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -13265,7 +13349,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.9.0 [[package]] name = "sp-std" version = "14.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" +source = "git+https://github.com/paritytech/polkadot-sdk#6dd777ffe62cff936e04a76134ccf07de9dee429" [[package]] name = "sp-storage" @@ -13283,7 +13367,7 @@ dependencies = [ [[package]] name = "sp-storage" version = "19.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" +source = "git+https://github.com/paritytech/polkadot-sdk#6dd777ffe62cff936e04a76134ccf07de9dee429" dependencies = [ "impl-serde", "parity-scale-codec", @@ -13347,7 +13431,7 @@ dependencies = [ [[package]] name = "sp-tracing" version = "16.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" +source = "git+https://github.com/paritytech/polkadot-sdk#6dd777ffe62cff936e04a76134ccf07de9dee429" dependencies = [ "parity-scale-codec", "tracing", @@ -13453,7 +13537,7 @@ dependencies = [ "parity-scale-codec", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -13486,7 +13570,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" version = "20.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" +source = "git+https://github.com/paritytech/polkadot-sdk#6dd777ffe62cff936e04a76134ccf07de9dee429" dependencies = [ "impl-trait-for-tuples", "log", @@ -13646,6 +13730,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.8.0" @@ -13729,7 +13824,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -13786,7 +13881,7 @@ name = "substrate-prometheus-endpoint" version = "0.17.0" source = "git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.9.0#3c3d6fceb82372a3019b37117aa453d564b212de" dependencies = [ - "hyper 0.14.29", + "hyper 0.14.30", "log", "prometheus", "thiserror", @@ -13860,7 +13955,7 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "subxt" version = "0.37.0" -source = "git+https://github.com/paritytech/subxt#b076f4c66ca2b7627f32d899211ea99a47db1c28" +source = "git+https://github.com/paritytech/subxt#f9d6f84bdd58e5e9ab0464e2052c8ed6a62b16ec" dependencies = [ "async-trait", "derive-where", @@ -13895,7 +13990,7 @@ dependencies = [ [[package]] name = "subxt-codegen" version = "0.37.0" -source = "git+https://github.com/paritytech/subxt#b076f4c66ca2b7627f32d899211ea99a47db1c28" +source = "git+https://github.com/paritytech/subxt#f9d6f84bdd58e5e9ab0464e2052c8ed6a62b16ec" dependencies = [ "frame-metadata 16.0.0", "heck 0.5.0", @@ -13907,7 +14002,7 @@ dependencies = [ "scale-info", "scale-typegen", "subxt-metadata", - "syn 2.0.68", + "syn 2.0.70", "thiserror", "tokio", ] @@ -13915,7 +14010,7 @@ dependencies = [ [[package]] name = "subxt-core" version = "0.37.0" -source = "git+https://github.com/paritytech/subxt#b076f4c66ca2b7627f32d899211ea99a47db1c28" +source = "git+https://github.com/paritytech/subxt#f9d6f84bdd58e5e9ab0464e2052c8ed6a62b16ec" dependencies = [ "base58", "blake2 0.10.6", @@ -13944,7 +14039,7 @@ dependencies = [ [[package]] name = "subxt-lightclient" version = "0.37.0" -source = "git+https://github.com/paritytech/subxt#b076f4c66ca2b7627f32d899211ea99a47db1c28" +source = "git+https://github.com/paritytech/subxt#f9d6f84bdd58e5e9ab0464e2052c8ed6a62b16ec" dependencies = [ "futures", "futures-util", @@ -13960,21 +14055,21 @@ dependencies = [ [[package]] name = "subxt-macro" version = "0.37.0" -source = "git+https://github.com/paritytech/subxt#b076f4c66ca2b7627f32d899211ea99a47db1c28" +source = "git+https://github.com/paritytech/subxt#f9d6f84bdd58e5e9ab0464e2052c8ed6a62b16ec" dependencies = [ - "darling 0.20.9", + "darling 0.20.10", "parity-scale-codec", "proc-macro-error", "quote", "scale-typegen", "subxt-codegen", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "subxt-metadata" version = "0.37.0" -source = "git+https://github.com/paritytech/subxt#b076f4c66ca2b7627f32d899211ea99a47db1c28" +source = "git+https://github.com/paritytech/subxt#f9d6f84bdd58e5e9ab0464e2052c8ed6a62b16ec" dependencies = [ "frame-metadata 16.0.0", "hashbrown 0.14.5", @@ -13996,9 +14091,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" dependencies = [ "proc-macro2", "quote", @@ -14072,9 +14167,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.14" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" +checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" [[package]] name = "tempfile" @@ -14173,7 +14268,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -14283,9 +14378,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6b6a2fb3a985e99cebfaefa9faa3024743da73304ca1c683a36429613d3d22" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -14334,7 +14429,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -14347,6 +14442,32 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-postgres" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot 0.12.3", + "percent-encoding", + "phf 0.11.2", + "pin-project-lite 0.2.14", + "postgres-protocol", + "postgres-types", + "rand 0.8.5", + "socket2 0.5.7", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-retry" version = "0.3.0" @@ -14385,7 +14506,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.10", + "rustls 0.23.11", "rustls-pki-types", "tokio", ] @@ -14458,7 +14579,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.14", + "toml_edit 0.22.15", ] [[package]] @@ -14496,9 +14617,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.14" +version = "0.22.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" dependencies = [ "indexmap 2.2.6", "serde", @@ -14521,7 +14642,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-timeout", "percent-encoding", "pin-project", @@ -14548,7 +14669,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-timeout", "percent-encoding", "pin-project", @@ -14635,7 +14756,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -14832,7 +14953,7 @@ version = "0.38.0" source = "git+https://github.com/paritytech/polkadot-sdk.git?tag=polkadot-v1.9.0#3c3d6fceb82372a3019b37117aa453d564b212de" dependencies = [ "async-trait", - "clap 4.5.8", + "clap 4.5.9", "frame-remote-externalities", "frame-try-runtime", "hex", @@ -14978,6 +15099,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -15098,9 +15225,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom 0.2.15", "serde", @@ -15242,6 +15369,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -15263,7 +15396,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-shared", ] @@ -15297,7 +15430,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -15683,6 +15816,17 @@ dependencies = [ "rustix 0.38.34", ] +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", + "web-sys", +] + [[package]] name = "wide" version = "0.7.25" @@ -16110,7 +16254,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -16130,7 +16274,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fb1bb144..c7e24ff2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,8 +59,8 @@ criterion = { version = "0.5.1", features = ["async_futures", "async_tokio"] } crossbeam = "^0.8" custom_error = "1.9.2" derivative = "2.2.0" -diesel = { version = "2.1", features = ["postgres", "uuid", "chrono", "r2d2"] } -diesel-async = { version = "0.4" } +diesel = { version = "^2.1", features = ["uuid", "chrono"] } +diesel-async = { version = "^0.4.1", features = ["postgres","deadpool","async-connection-wrapper","tokio"]} diesel_migrations = { version = "2.1", features = ["postgres"] } dotenvy = "0.15" frame-benchmarking = "24.0.0" diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index b20c8ed8..869f78a3 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -24,6 +24,7 @@ chrono = { workspace = true } custom_error = { workspace = true } derivative = { workspace = true } diesel = { workspace = true } +diesel-async = { workspace = true } diesel_migrations = { workspace = true } futures = { workspace = true } hex = { workspace = true } diff --git a/crates/api/src/api.rs b/crates/api/src/api.rs index 8226137d..d71f9eb5 100644 --- a/crates/api/src/api.rs +++ b/crates/api/src/api.rs @@ -1,21 +1,15 @@ -use std::{ - marker::PhantomData, - time::{Duration, Instant}, -}; - use async_graphql::futures_util::select; use chrono::{DateTime, Utc}; -use diesel::{r2d2::ConnectionManager, PgConnection}; -use diesel_migrations::MigrationHarness; +use diesel_async::{pooled_connection::deadpool::Pool, AsyncPgConnection}; use futures::{FutureExt, StreamExt}; use metrics::histogram; use metrics_exporter_prometheus::PrometheusBuilder; -use r2d2::Pool; +use std::time::{Duration, Instant}; use tokio::sync::{broadcast::Sender, mpsc, mpsc::Receiver}; use tracing::{debug, error, info, info_span, instrument, trace, warn, Instrument}; use uuid::Uuid; -use chronicle_persistence::{Store, StoreError, MIGRATIONS}; +use chronicle_persistence::Store; use chronicle_signing::ChronicleSigning; use common::{ attributes::Attributes, @@ -51,7 +45,6 @@ use crate::{ #[derive(Clone)] pub struct Api< - U: UuidGen + Send + Sync + Clone, W: LedgerWriter + Clone + Send @@ -62,12 +55,10 @@ pub struct Api< signing: ChronicleSigning, ledger_writer: W, store: Store, - uuid_source: PhantomData, } -impl Api +impl Api where - U: UuidGen + Send + Sync + Clone + core::fmt::Debug + 'static, LEDGER: LedgerWriter + LedgerReader + Clone @@ -75,11 +66,9 @@ where + Sync + 'static, { - #[instrument(skip(ledger), name = "api_new")] - pub async fn new( - pool: Pool>, + pub async fn create_dispatch( + pool: Pool, ledger: LEDGER, - uuidgen: U, signing: ChronicleSigning, namespace_bindings: Vec, policy_address: Option, @@ -93,27 +82,26 @@ where let store = Store::new(pool.clone())?; - pool.get()? - .build_transaction() - .run(|connection| connection.run_pending_migrations(MIGRATIONS).map(|_| ())) - .map_err(StoreError::DbMigration)?; - let system_namespace_uuid = (SYSTEM_ID, Uuid::try_from(SYSTEM_UUID).unwrap()); // Append namespace bindings and system namespace - store.namespace_binding(system_namespace_uuid.0, system_namespace_uuid.1)?; + store + .namespace_binding(system_namespace_uuid.0, system_namespace_uuid.1) + .await?; for ns in namespace_bindings { info!( "Binding namespace with external ID: {}, UUID: {}", ns.external_id_part().as_str(), ns.uuid_part() ); - store.namespace_binding(ns.external_id_part().as_str(), ns.uuid_part().to_owned())? + store + .namespace_binding(ns.external_id_part().as_str(), ns.uuid_part().to_owned()) + .await?; } let reuse_reader = ledger.clone(); - let last_seen_block = store.get_last_block_id(); + let last_seen_block = store.get_last_block_id().await; let start_from_block = if let Ok(Some(start_from_block)) = last_seen_block { FromBlock::BlockId(start_from_block) @@ -236,12 +224,11 @@ where start_from_block: FromBlock, ) { tokio::task::spawn(async move { - let mut api = Api:: { + let api = Api:: { submit_tx: commit_notify_tx.clone(), signing, ledger_writer: ledger, store: store.clone(), - uuid_source: PhantomData, }; loop { @@ -318,17 +305,16 @@ where /// Notify after a successful submission, depending on the consistency requirement TODO: set in /// the transaction - fn submit_blocking( - &mut self, + async fn submit_transaction( + &self, tx: ChronicleTransaction, ) -> Result { - let (submission, _id) = futures::executor::block_on(self.ledger_writer.pre_submit(tx))?; + let (submission, _id) = self.ledger_writer.pre_submit(tx).await?; - let res = - futures::executor::block_on(self.ledger_writer.do_submit( - protocol_substrate_chronicle::protocol::WriteConsistency::Weak, - submission, - )); + let res = self + .ledger_writer + .do_submit(protocol_substrate_chronicle::protocol::WriteConsistency::Weak, submission) + .await; match res { Ok(tx_id) => { self.submit_tx.send(SubmissionStage::submitted(&tx_id)).ok(); @@ -345,17 +331,17 @@ where /// Generate and submit the signed identity to send to the Transaction Processor along with the /// transactions to be applied - fn submit( - &mut self, + async fn submit( + &self, id: impl Into, identity: AuthId, to_apply: Vec, ) -> Result { let identity = identity.signed_identity(&self.signing)?; let model = ProvModel::from_tx(&to_apply).map_err(ApiError::Contradiction)?; - let tx_id = self.submit_blocking(futures::executor::block_on( - ChronicleTransaction::new(&self.signing, identity, to_apply), - )?)?; + let tx_id = self + .submit_transaction(ChronicleTransaction::new(&self.signing, identity, to_apply).await?) + .await?; Ok(ApiResponse::submission(id, model, tx_id)) } @@ -367,9 +353,9 @@ where /// * `connection` - Connection to the Chronicle database /// * `to_apply` - Chronicle operations resulting from an API call #[instrument(skip(self, connection, to_apply))] - fn check_for_effects( - &mut self, - connection: &mut PgConnection, + async fn check_for_effects( + &self, + connection: &mut AsyncPgConnection, to_apply: &Vec, ) -> Result>, ApiError> { debug!(checking_for_effects = to_apply.len()); @@ -379,96 +365,120 @@ where let mut applied_model = match op { ChronicleOperation::CreateNamespace(CreateNamespace { id, .. }) => { let (namespace, _) = - self.ensure_namespace(connection, id.external_id_part())?; + self.ensure_namespace(connection, id.external_id_part()).await?; model.namespace_context(&namespace); model }, ChronicleOperation::AgentExists(AgentExists { ref namespace, ref id }) => - self.store.apply_prov_model_for_agent_id( - connection, - model, - id, - namespace.external_id_part(), - )?, + self.store + .apply_prov_model_for_agent_id( + connection, + model, + id, + namespace.external_id_part(), + ) + .await?, ChronicleOperation::ActivityExists(ActivityExists { ref namespace, ref id }) => - self.store.apply_prov_model_for_activity_id( - connection, - model, - id, - namespace.external_id_part(), - )?, + self.store + .apply_prov_model_for_activity_id( + connection, + model, + id, + namespace.external_id_part(), + ) + .await?, ChronicleOperation::EntityExists(EntityExists { ref namespace, ref id }) => - self.store.apply_prov_model_for_entity_id( - connection, - model, - id, - namespace.external_id_part(), - )?, + self.store + .apply_prov_model_for_entity_id( + connection, + model, + id, + namespace.external_id_part(), + ) + .await?, ChronicleOperation::ActivityUses(ActivityUses { ref namespace, ref id, ref activity, - }) => self.store.prov_model_for_usage( - connection, - model, - id, - activity, - namespace.external_id_part(), - )?, - ChronicleOperation::SetAttributes(ref o) => match o { - SetAttributes::Activity { namespace, id, .. } => - self.store.apply_prov_model_for_activity_id( + }) => + self.store + .prov_model_for_usage( connection, model, id, + activity, namespace.external_id_part(), - )?, + ) + .await?, + ChronicleOperation::SetAttributes(ref o) => match o { + SetAttributes::Activity { namespace, id, .. } => + self.store + .apply_prov_model_for_activity_id( + connection, + model, + id, + namespace.external_id_part(), + ) + .await?, SetAttributes::Agent { namespace, id, .. } => - self.store.apply_prov_model_for_agent_id( + self.store + .apply_prov_model_for_agent_id( + connection, + model, + id, + namespace.external_id_part(), + ) + .await?, + SetAttributes::Entity { namespace, id, .. } => + self.store + .apply_prov_model_for_entity_id( + connection, + model, + id, + namespace.external_id_part(), + ) + .await?, + }, + ChronicleOperation::StartActivity(StartActivity { namespace, id, .. }) => + self.store + .apply_prov_model_for_activity_id( connection, model, id, namespace.external_id_part(), - )?, - SetAttributes::Entity { namespace, id, .. } => - self.store.apply_prov_model_for_entity_id( + ) + .await?, + ChronicleOperation::EndActivity(EndActivity { namespace, id, .. }) => + self.store + .apply_prov_model_for_activity_id( connection, model, id, namespace.external_id_part(), - )?, - }, - ChronicleOperation::StartActivity(StartActivity { namespace, id, .. }) => - self.store.apply_prov_model_for_activity_id( - connection, - model, - id, - namespace.external_id_part(), - )?, - ChronicleOperation::EndActivity(EndActivity { namespace, id, .. }) => - self.store.apply_prov_model_for_activity_id( - connection, - model, - id, - namespace.external_id_part(), - )?, + ) + .await?, ChronicleOperation::WasInformedBy(WasInformedBy { namespace, activity, informing_activity, }) => { - let model = self.store.apply_prov_model_for_activity_id( - connection, - model, - activity, - namespace.external_id_part(), - )?; - self.store.apply_prov_model_for_activity_id( - connection, - model, - informing_activity, - namespace.external_id_part(), - )? + let model = self + .store + .apply_prov_model_for_activity_id( + connection, + model, + activity, + namespace.external_id_part(), + ) + .await?; + self.store + .apply_prov_model_for_activity_id( + connection, + model, + informing_activity, + namespace.external_id_part(), + ) + .await? }, ChronicleOperation::AgentActsOnBehalfOf(ActsOnBehalfOf { activity_id, @@ -477,25 +487,33 @@ where namespace, .. }) => { - let model = self.store.apply_prov_model_for_agent_id( - connection, - model, - responsible_id, - namespace.external_id_part(), - )?; - let model = self.store.apply_prov_model_for_agent_id( - connection, - model, - delegate_id, - namespace.external_id_part(), - )?; - if let Some(id) = activity_id { - self.store.apply_prov_model_for_activity_id( + let model = self + .store + .apply_prov_model_for_agent_id( connection, model, - id, + responsible_id, namespace.external_id_part(), - )? + ) + .await?; + let model = self + .store + .apply_prov_model_for_agent_id( + connection, + model, + delegate_id, + namespace.external_id_part(), + ) + .await?; + if let Some(id) = activity_id { + self.store + .apply_prov_model_for_activity_id( + connection, + model, + id, + namespace.external_id_part(), + ) + .await? } else { model } @@ -506,34 +524,44 @@ where agent_id, .. }) => { - let model = self.store.apply_prov_model_for_activity_id( - connection, - model, - activity_id, - namespace.external_id_part(), - )?; + let model = self + .store + .apply_prov_model_for_activity_id( + connection, + model, + activity_id, + namespace.external_id_part(), + ) + .await?; - self.store.apply_prov_model_for_agent_id( - connection, - model, - agent_id, - namespace.external_id_part(), - )? + self.store + .apply_prov_model_for_agent_id( + connection, + model, + agent_id, + namespace.external_id_part(), + ) + .await? }, ChronicleOperation::WasGeneratedBy(WasGeneratedBy { namespace, id, activity }) => { - let model = self.store.apply_prov_model_for_activity_id( - connection, - model, - activity, - namespace.external_id_part(), - )?; + let model = self + .store + .apply_prov_model_for_activity_id( + connection, + model, + activity, + namespace.external_id_part(), + ) + .await?; - self.store.apply_prov_model_for_entity_id( - connection, - model, - id, - namespace.external_id_part(), - )? + self.store + .apply_prov_model_for_entity_id( + connection, + model, + id, + namespace.external_id_part(), + ) + .await? }, ChronicleOperation::EntityDerive(EntityDerive { namespace, @@ -542,27 +570,35 @@ where activity_id, .. }) => { - let model = self.store.apply_prov_model_for_entity_id( - connection, - model, - id, - namespace.external_id_part(), - )?; - - let model = self.store.apply_prov_model_for_entity_id( - connection, - model, - used_id, - namespace.external_id_part(), - )?; - - if let Some(id) = activity_id { - self.store.apply_prov_model_for_activity_id( + let model = self + .store + .apply_prov_model_for_entity_id( connection, model, id, namespace.external_id_part(), - )? + ) + .await?; + + let model = self + .store + .apply_prov_model_for_entity_id( + connection, + model, + used_id, + namespace.external_id_part(), + ) + .await?; + + if let Some(id) = activity_id { + self.store + .apply_prov_model_for_activity_id( + connection, + model, + id, + namespace.external_id_part(), + ) + .await? } else { model } @@ -573,19 +609,24 @@ where agent_id, .. }) => { - let model = self.store.apply_prov_model_for_entity_id( - connection, - model, - entity_id, - namespace.external_id_part(), - )?; + let model = self + .store + .apply_prov_model_for_entity_id( + connection, + model, + entity_id, + namespace.external_id_part(), + ) + .await?; - self.store.apply_prov_model_for_agent_id( - connection, - model, - agent_id, - namespace.external_id_part(), - )? + self.store + .apply_prov_model_for_agent_id( + connection, + model, + agent_id, + namespace.external_id_part(), + ) + .await? }, }; let state = applied_model.clone(); @@ -604,19 +645,19 @@ where } } - fn apply_effects_and_submit( - &mut self, - connection: &mut PgConnection, + async fn apply_effects_and_submit( + &self, + connection: &mut AsyncPgConnection, id: impl Into, identity: AuthId, to_apply: Vec, applying_new_namespace: bool, ) -> Result { if applying_new_namespace { - self.submit(id, identity, to_apply) - } else if let Some(to_apply) = self.check_for_effects(connection, &to_apply)? { + self.submit(id, identity, to_apply).await + } else if let Some(to_apply) = self.check_for_effects(connection, &to_apply).await? { info!(sending_operations = to_apply.len()); - self.submit(id, identity, to_apply) + self.submit(id, identity, to_apply).await } else { info!("API call will not result in any data changes"); let model = ProvModel::from_tx(&to_apply)?; @@ -633,12 +674,12 @@ where /// binding operation to tie the UUID from another instance to an external_id # Arguments /// * `external_id` - an arbitrary namespace identifier #[instrument(skip(self, connection))] - fn ensure_namespace( - &mut self, - connection: &mut PgConnection, + async fn ensure_namespace( + &self, + connection: &mut AsyncPgConnection, id: &ExternalId, ) -> Result<(NamespaceId, Vec), ApiError> { - match self.store.namespace_by_external_id(connection, id) { + match self.store.namespace_by_external_id(connection, id).await { Ok((namespace_id, _)) => { trace!(%id, "Namespace already exists."); Ok((namespace_id, vec![])) @@ -654,11 +695,6 @@ where } } - /// Creates and submits a (ChronicleTransaction::GenerateEntity), and possibly - /// (ChronicleTransaction::Domaintype) if specified - /// - /// We use our local store for the best guess at the activity, either by external_id or the last - /// one started as a convenience for command line #[instrument(skip(self))] async fn activity_generate( &self, @@ -667,38 +703,38 @@ where activity_id: ActivityId, identity: AuthId, ) -> Result { - let mut api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; + let mut connection = self.store.connection().await?; + let api = self.clone(); + connection + .build_transaction() + .run(|connection| { + Box::pin(async move { + let (namespace, mut to_apply) = + api.ensure_namespace(connection, &namespace).await?; - let applying_new_namespace = !to_apply.is_empty(); + let applying_new_namespace = !to_apply.is_empty(); - let create = ChronicleOperation::WasGeneratedBy(WasGeneratedBy { - namespace, - id: id.clone(), - activity: activity_id, - }); + let create = ChronicleOperation::WasGeneratedBy(WasGeneratedBy { + namespace, + id: id.clone(), + activity: activity_id, + }); - to_apply.push(create); + to_apply.push(create); - api.apply_effects_and_submit( - connection, - id, - identity, - to_apply, - applying_new_namespace, - ) + api.apply_effects_and_submit( + connection, + id, + identity, + to_apply, + applying_new_namespace, + ) + .await + }) }) - }) - .await? + .await } - /// Creates and submits a (ChronicleTransaction::ActivityUses), and possibly - /// (ChronicleTransaction::Domaintype) if specified We use our local store for the best guess at - /// the activity, either by name or the last one started as a convenience for command line #[instrument(skip(self))] async fn activity_use( &self, @@ -707,37 +743,40 @@ where activity_id: ActivityId, identity: AuthId, ) -> Result { - let mut api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; + let mut connection = self.store.connection().await?; + let api = self.clone(); + connection + .build_transaction() + .run(|connection| { + Box::pin(async move { + let (namespace, mut to_apply) = + api.ensure_namespace(connection, &namespace).await?; - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; + let applying_new_namespace = !to_apply.is_empty(); - let applying_new_namespace = !to_apply.is_empty(); + let (id, to_apply) = { + let create = ChronicleOperation::ActivityUses(ActivityUses { + namespace, + id: id.clone(), + activity: activity_id, + }); - let (id, to_apply) = { - let create = ChronicleOperation::ActivityUses(ActivityUses { - namespace, - id: id.clone(), - activity: activity_id, - }); + to_apply.push(create); - to_apply.push(create); + (id, to_apply) + }; - (id, to_apply) - }; - - api.apply_effects_and_submit( - connection, - id, - identity, - to_apply, - applying_new_namespace, - ) + api.apply_effects_and_submit( + connection, + id, + identity, + to_apply, + applying_new_namespace, + ) + .await + }) }) - }) - .await? + .await } /// Creates and submits a (ChronicleTransaction::ActivityWasInformedBy) @@ -752,37 +791,40 @@ where informing_activity_id: ActivityId, identity: AuthId, ) -> Result { - let mut api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; + let mut connection = self.store.connection().await?; + let api = self.clone(); + connection + .build_transaction() + .run(|connection| { + Box::pin(async move { + let (namespace, mut to_apply) = + api.ensure_namespace(connection, &namespace).await?; - let applying_new_namespace = !to_apply.is_empty(); + let applying_new_namespace = !to_apply.is_empty(); - let (id, to_apply) = { - let create = ChronicleOperation::WasInformedBy(WasInformedBy { - namespace, - activity: id.clone(), - informing_activity: informing_activity_id, - }); + let (id, to_apply) = { + let create = ChronicleOperation::WasInformedBy(WasInformedBy { + namespace, + activity: id.clone(), + informing_activity: informing_activity_id, + }); - to_apply.push(create); + to_apply.push(create); - (id, to_apply) - }; + (id, to_apply) + }; - api.apply_effects_and_submit( - connection, - id, - identity, - to_apply, - applying_new_namespace, - ) + api.apply_effects_and_submit( + connection, + id, + identity, + to_apply, + applying_new_namespace, + ) + .await + }) }) - }) - .await? + .await } /// Submits operations [`CreateEntity`], and [`SetAttributes::Entity`] @@ -796,40 +838,42 @@ where attributes: Attributes, identity: AuthId, ) -> Result { - let mut api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace_id)?; - - let applying_new_namespace = !to_apply.is_empty(); - - let create = ChronicleOperation::EntityExists(EntityExists { - namespace: namespace.clone(), - id: id.clone(), - }); + let mut connection = self.store.connection().await?; + let api = self.clone(); + connection + .build_transaction() + .run(|connection| { + Box::pin(async move { + let (namespace, mut to_apply) = + api.ensure_namespace(connection, &namespace_id).await?; + let applying_new_namespace = !to_apply.is_empty(); + + let create = ChronicleOperation::EntityExists(EntityExists { + namespace: namespace.clone(), + id: id.clone(), + }); - to_apply.push(create); + to_apply.push(create); - let set_type = ChronicleOperation::SetAttributes(SetAttributes::Entity { - id: id.clone(), - namespace, - attributes, - }); + let set_type = ChronicleOperation::SetAttributes(SetAttributes::Entity { + id: id.clone(), + namespace, + attributes, + }); - to_apply.push(set_type); + to_apply.push(set_type); - api.apply_effects_and_submit( - connection, - id, - identity, - to_apply, - applying_new_namespace, - ) + api.apply_effects_and_submit( + connection, + id, + identity, + to_apply, + applying_new_namespace, + ) + .await + }) }) - }) - .await? + .await } /// Submits operations [`CreateActivity`], and [`SetAttributes::Activity`] @@ -843,40 +887,42 @@ where attributes: Attributes, identity: AuthId, ) -> Result { - let mut api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace_id)?; - - let applying_new_namespace = !to_apply.is_empty(); - - let create = ChronicleOperation::ActivityExists(ActivityExists { - namespace: namespace.clone(), - id: ActivityId::from_external_id(&activity_id), - }); + let mut connection = self.store.connection().await?; + let api = self.clone(); + connection + .build_transaction() + .run(|connection| { + Box::pin(async move { + let (namespace, mut to_apply) = + api.ensure_namespace(connection, &namespace_id).await?; + let applying_new_namespace = !to_apply.is_empty(); + + let create = ChronicleOperation::ActivityExists(ActivityExists { + namespace: namespace.clone(), + id: ActivityId::from_external_id(&activity_id), + }); - to_apply.push(create); + to_apply.push(create); - let set_type = ChronicleOperation::SetAttributes(SetAttributes::Activity { - id: ActivityId::from_external_id(&activity_id), - namespace, - attributes, - }); + let set_type = ChronicleOperation::SetAttributes(SetAttributes::Activity { + id: ActivityId::from_external_id(&activity_id), + namespace, + attributes, + }); - to_apply.push(set_type); + to_apply.push(set_type); - api.apply_effects_and_submit( - connection, - ActivityId::from_external_id(&activity_id), - identity, - to_apply, - applying_new_namespace, - ) + api.apply_effects_and_submit( + connection, + ActivityId::from_external_id(&activity_id), + identity, + to_apply, + applying_new_namespace, + ) + .await + }) }) - }) - .await? + .await } /// Submits operations [`CreateAgent`], and [`SetAttributes::Agent`] @@ -890,41 +936,43 @@ where attributes: Attributes, identity: AuthId, ) -> Result { - let mut api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; - - let applying_new_namespace = !to_apply.is_empty(); - - let create = ChronicleOperation::AgentExists(AgentExists { - id: AgentId::from_external_id(&agent_id), - namespace: namespace.clone(), - }); + let mut connection = self.store.connection().await?; + let api = self.clone(); + connection + .build_transaction() + .run(|connection| { + Box::pin(async move { + let (namespace, mut to_apply) = + api.ensure_namespace(connection, &namespace).await?; + let applying_new_namespace = !to_apply.is_empty(); + + let create = ChronicleOperation::AgentExists(AgentExists { + id: AgentId::from_external_id(&agent_id), + namespace: namespace.clone(), + }); - to_apply.push(create); + to_apply.push(create); - let id = AgentId::from_external_id(&agent_id); - let set_type = ChronicleOperation::SetAttributes(SetAttributes::Agent { - id: id.clone(), - namespace, - attributes, - }); + let id = AgentId::from_external_id(&agent_id); + let set_type = ChronicleOperation::SetAttributes(SetAttributes::Agent { + id: id.clone(), + namespace, + attributes, + }); - to_apply.push(set_type); + to_apply.push(set_type); - api.apply_effects_and_submit( - connection, - id, - identity, - to_apply, - applying_new_namespace, - ) + api.apply_effects_and_submit( + connection, + id, + identity, + to_apply, + applying_new_namespace, + ) + .await + }) }) - }) - .await? + .await } /// Creates and submits a (ChronicleTransaction::CreateNamespace) if the external_id part does @@ -934,17 +982,16 @@ where name: &ExternalId, identity: AuthId, ) -> Result { - let mut api = self.clone(); - let name = name.to_owned(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - connection.build_transaction().run(|connection| { - let (namespace, to_apply) = api.ensure_namespace(connection, &name)?; - - api.submit(namespace, identity, to_apply) + let mut connection = self.store.connection().await?; + connection + .build_transaction() + .run(|connection| { + Box::pin(async move { + let (namespace, to_apply) = self.ensure_namespace(connection, name).await?; + self.submit(namespace, identity, to_apply).await + }) }) - }) - .await? + .await } #[instrument(skip(self))] @@ -953,40 +1000,32 @@ where namespace: NamespaceId, identity: AuthId, ) -> Result { - let mut api = self.clone(); let id = ActivityId::from_external_id(Uuid::new_v4().to_string()); - tokio::task::spawn_blocking(move || { - let to_apply = vec![ - ChronicleOperation::StartActivity(StartActivity { - namespace: namespace.clone(), - id: id.clone(), - time: Utc::now().into(), - }), - ChronicleOperation::EndActivity(EndActivity { - namespace, - id, - time: Utc::now().into(), - }), - ]; - api.submit_depth_charge(identity, to_apply) - }) - .await? + let to_apply = vec![ + ChronicleOperation::StartActivity(StartActivity { + namespace: namespace.clone(), + id: id.clone(), + time: Utc::now().into(), + }), + ChronicleOperation::EndActivity(EndActivity { namespace, id, time: Utc::now().into() }), + ]; + self.submit_depth_charge(identity, to_apply).await } - fn submit_depth_charge( - &mut self, + pub async fn submit_depth_charge( + &self, identity: AuthId, to_apply: Vec, ) -> Result { let identity = identity.signed_identity(&self.signing)?; - let tx_id = self.submit_blocking(futures::executor::block_on( - ChronicleTransaction::new(&self.signing, identity, to_apply), - )?)?; + let tx_id = self + .submit_transaction(ChronicleTransaction::new(&self.signing, identity, to_apply).await?) + .await?; Ok(ApiResponse::depth_charge_submission(tx_id)) } #[instrument(skip(self))] - async fn dispatch(&mut self, command: (ApiCommand, AuthId)) -> Result { + pub async fn dispatch(&self, command: (ApiCommand, AuthId)) -> Result { match command { (ApiCommand::DepthCharge(DepthChargeCommand { namespace }), identity) => self.depth_charge(namespace, identity).await, @@ -996,8 +1035,6 @@ where self.create_namespace(&id, identity).await, (ApiCommand::Agent(AgentCommand::Create { id, namespace, attributes }), identity) => self.create_agent(id, namespace, attributes, identity).await, - (ApiCommand::Agent(AgentCommand::UseInContext { id, namespace }), _identity) => - self.use_agent_in_cli_context(id, namespace).await, ( ApiCommand::Agent(AgentCommand::Delegate { id, @@ -1080,36 +1117,36 @@ where role: Option, identity: AuthId, ) -> Result { - let mut api = self.clone(); - - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; - - let applying_new_namespace = !to_apply.is_empty(); + let mut connection = self.store.connection().await?; + connection + .build_transaction() + .run(|connection| { + Box::pin(async move { + let (namespace, mut to_apply) = + self.ensure_namespace(connection, &namespace).await?; + let applying_new_namespace = !to_apply.is_empty(); - let tx = ChronicleOperation::agent_acts_on_behalf_of( - namespace, - responsible_id.clone(), - delegate_id, - activity_id, - role, - ); + let tx = ChronicleOperation::agent_acts_on_behalf_of( + namespace, + responsible_id.clone(), + delegate_id, + activity_id, + role, + ); - to_apply.push(tx); + to_apply.push(tx); - api.apply_effects_and_submit( - connection, - responsible_id, - identity, - to_apply, - applying_new_namespace, - ) + self.apply_effects_and_submit( + connection, + responsible_id, + identity, + to_apply, + applying_new_namespace, + ) + .await + }) }) - }) - .await? + .await } #[instrument(skip(self))] @@ -1121,35 +1158,35 @@ where role: Option, identity: AuthId, ) -> Result { - let mut api = self.clone(); - - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; - - let applying_new_namespace = !to_apply.is_empty(); + let mut connection = self.store.connection().await?; + connection + .build_transaction() + .run(|connection| { + Box::pin(async move { + let (namespace, mut to_apply) = + self.ensure_namespace(connection, &namespace).await?; + let applying_new_namespace = !to_apply.is_empty(); - let tx = ChronicleOperation::was_associated_with( - namespace, - activity_id, - responsible_id.clone(), - role, - ); + let tx = ChronicleOperation::was_associated_with( + namespace, + activity_id, + responsible_id.clone(), + role, + ); - to_apply.push(tx); + to_apply.push(tx); - api.apply_effects_and_submit( - connection, - responsible_id, - identity, - to_apply, - applying_new_namespace, - ) + self.apply_effects_and_submit( + connection, + responsible_id, + identity, + to_apply, + applying_new_namespace, + ) + .await + }) }) - }) - .await? + .await } #[instrument(skip(self))] @@ -1161,35 +1198,27 @@ where role: Option, identity: AuthId, ) -> Result { - let mut api = self.clone(); - - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; - - let applying_new_namespace = !to_apply.is_empty(); - - let tx = ChronicleOperation::was_attributed_to( - namespace, - entity_id, - responsible_id.clone(), - role, - ); + let mut connection = self.store.connection().await?; + let (namespace, mut to_apply) = self.ensure_namespace(&mut connection, &namespace).await?; + let applying_new_namespace = !to_apply.is_empty(); + + let tx = ChronicleOperation::was_attributed_to( + namespace, + entity_id, + responsible_id.clone(), + role, + ); - to_apply.push(tx); + to_apply.push(tx); - api.apply_effects_and_submit( - connection, - responsible_id, - identity, - to_apply, - applying_new_namespace, - ) - }) - }) - .await? + self.apply_effects_and_submit( + &mut connection, + responsible_id, + identity, + to_apply, + applying_new_namespace, + ) + .await } #[instrument(skip(self))] @@ -1202,51 +1231,39 @@ where typ: DerivationType, identity: AuthId, ) -> Result { - let mut api = self.clone(); - - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; - - let applying_new_namespace = !to_apply.is_empty(); - - let tx = ChronicleOperation::EntityDerive(EntityDerive { - namespace, - id: id.clone(), - used_id: used_id.clone(), - activity_id: activity_id.clone(), - typ, - }); + let mut connection = self.store.connection().await?; + let (namespace, mut to_apply) = self.ensure_namespace(&mut connection, &namespace).await?; + let applying_new_namespace = !to_apply.is_empty(); + + let tx = ChronicleOperation::EntityDerive(EntityDerive { + namespace, + id: id.clone(), + used_id: used_id.clone(), + activity_id: activity_id.clone(), + typ, + }); - to_apply.push(tx); + to_apply.push(tx); - api.apply_effects_and_submit( - connection, - id, - identity, - to_apply, - applying_new_namespace, - ) - }) - }) - .await? + self.apply_effects_and_submit( + &mut connection, + id, + identity, + to_apply, + applying_new_namespace, + ) + .await } async fn query(&self, query: QueryCommand) -> Result { - let api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - let (id, _) = api - .store - .namespace_by_external_id(&mut connection, &ExternalId::from(&query.namespace))?; - Ok(ApiResponse::query_reply( - api.store.load_prov_model_for_namespace(&mut connection, &id)?, - )) - }) - .await? + let mut connection = self.store.connection().await?; + let (id, _) = self + .store + .namespace_by_external_id(&mut connection, &ExternalId::from(&query.namespace)) + .await?; + Ok(ApiResponse::query_reply( + self.store.load_prov_model_for_namespace(&mut connection, &id).await?, + )) } async fn submit_import_operations( @@ -1254,49 +1271,38 @@ where identity: AuthId, operations: Vec, ) -> Result { - let mut api = self.clone(); let identity = identity.signed_identity(&self.signing)?; let model = ProvModel::from_tx(&operations)?; let signer = self.signing.clone(); - tokio::task::spawn_blocking(move || { - // Check here to ensure that import operations result in data changes - let mut connection = api.store.connection()?; - connection.build_transaction().run(|connection| { - if let Some(operations_to_apply) = api.check_for_effects(connection, &operations)? { - trace!( - operations_to_apply = operations_to_apply.len(), - "Import operations submitted" - ); - let tx_id = api.submit_blocking(futures::executor::block_on( - ChronicleTransaction::new(&signer, identity, operations_to_apply), - )?)?; - Ok(ApiResponse::import_submitted(model, tx_id)) - } else { - info!("Import will not result in any data changes"); - Ok(ApiResponse::AlreadyRecordedAll) - } - }) - }) - .await? + let mut connection = self.store.connection().await?; + if let Some(operations_to_apply) = + self.check_for_effects(&mut connection, &operations).await? + { + trace!(operations_to_apply = operations_to_apply.len(), "Import operations submitted"); + let tx_id = self + .submit_transaction( + ChronicleTransaction::new(&signer, identity, operations_to_apply).await?, + ) + .await?; + Ok(ApiResponse::import_submitted(model, tx_id)) + } else { + info!("Import will not result in any data changes"); + Ok(ApiResponse::AlreadyRecordedAll) + } } #[instrument(skip(self, prov), ret(Debug))] - async fn sync( + pub async fn sync( &self, prov: Box, block_id: &BlockId, tx_id: ChronicleTransactionId, ) -> Result { trace!(prov = ?prov); - let api = self.clone(); let block_id = *block_id; - tokio::task::spawn_blocking(move || { - api.store.apply_prov(&prov)?; - api.store.set_last_block_id(&block_id, tx_id)?; - - Ok(ApiResponse::Unit) - }) - .await? + self.store.apply_prov(&prov).await?; + self.store.set_last_block_id(&block_id, tx_id).await?; + Ok(ApiResponse::Unit) } /// Creates and submits a (ChronicleTransaction::StartActivity) determining the appropriate @@ -1307,61 +1313,44 @@ where id: ActivityId, namespace: ExternalId, time: Option>, - agent: Option, + agent_id: Option, identity: AuthId, ) -> Result { - let mut api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; - - let applying_new_namespace = !to_apply.is_empty(); - - let agent_id = { - if let Some(agent) = agent { - Some(agent) - } else { - api.store - .get_current_agent(connection) - .ok() - .map(|x| AgentId::from_external_id(x.external_id)) - } - }; - - let now = Utc::now(); - - to_apply.push(ChronicleOperation::StartActivity(StartActivity { - namespace: namespace.clone(), - id: id.clone(), - time: time.unwrap_or(now).into(), - })); - - to_apply.push(ChronicleOperation::EndActivity(EndActivity { - namespace: namespace.clone(), - id: id.clone(), - time: time.unwrap_or(now).into(), - })); - - if let Some(agent_id) = agent_id { - to_apply.push(ChronicleOperation::was_associated_with( - namespace, - id.clone(), - agent_id, - None, - )); - } + let mut connection = self.store.connection().await?; + let (namespace, mut to_apply) = self.ensure_namespace(&mut connection, &namespace).await?; + let applying_new_namespace = !to_apply.is_empty(); + + let now = Utc::now(); + + to_apply.push(ChronicleOperation::StartActivity(StartActivity { + namespace: namespace.clone(), + id: id.clone(), + time: time.unwrap_or(now).into(), + })); + + to_apply.push(ChronicleOperation::EndActivity(EndActivity { + namespace: namespace.clone(), + id: id.clone(), + time: time.unwrap_or(now).into(), + })); + + if let Some(agent_id) = agent_id { + to_apply.push(ChronicleOperation::was_associated_with( + namespace, + id.clone(), + agent_id, + None, + )); + } - api.apply_effects_and_submit( - connection, - id, - identity, - to_apply, - applying_new_namespace, - ) - }) - }) - .await? + self.apply_effects_and_submit( + &mut connection, + id, + identity, + to_apply, + applying_new_namespace, + ) + .await } /// Creates and submits a (ChronicleTransaction::StartActivity), determining the appropriate @@ -1372,53 +1361,36 @@ where id: ActivityId, namespace: ExternalId, time: Option>, - agent: Option, + agent_id: Option, identity: AuthId, ) -> Result { - let mut api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; - - let applying_new_namespace = !to_apply.is_empty(); - - let agent_id = { - if let Some(agent) = agent { - Some(agent) - } else { - api.store - .get_current_agent(connection) - .ok() - .map(|x| AgentId::from_external_id(x.external_id)) - } - }; - - to_apply.push(ChronicleOperation::StartActivity(StartActivity { - namespace: namespace.clone(), - id: id.clone(), - time: time.unwrap_or_else(Utc::now).into(), - })); - - if let Some(agent_id) = agent_id { - to_apply.push(ChronicleOperation::was_associated_with( - namespace, - id.clone(), - agent_id, - None, - )); - } + let mut connection = self.store.connection().await?; + let (namespace, mut to_apply) = self.ensure_namespace(&mut connection, &namespace).await?; + let applying_new_namespace = !to_apply.is_empty(); + + to_apply.push(ChronicleOperation::StartActivity(StartActivity { + namespace: namespace.clone(), + id: id.clone(), + time: time.unwrap_or_else(Utc::now).into(), + })); + + if let Some(agent_id) = agent_id { + to_apply.push(ChronicleOperation::was_associated_with( + namespace, + id.clone(), + agent_id, + None, + )); + } - api.apply_effects_and_submit( - connection, - id, - identity, - to_apply, - applying_new_namespace, - ) - }) - }) - .await? + self.apply_effects_and_submit( + &mut connection, + id, + identity, + to_apply, + applying_new_namespace, + ) + .await } /// Creates and submits a (ChronicleTransaction::EndActivity), determining the appropriate agent @@ -1429,72 +1401,36 @@ where id: ActivityId, namespace: ExternalId, time: Option>, - agent: Option, + agent_id: Option, identity: AuthId, ) -> Result { - let mut api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - connection.build_transaction().run(|connection| { - let (namespace, mut to_apply) = api.ensure_namespace(connection, &namespace)?; - - let applying_new_namespace = !to_apply.is_empty(); - - let agent_id = { - if let Some(agent) = agent { - Some(agent) - } else { - api.store - .get_current_agent(connection) - .ok() - .map(|x| AgentId::from_external_id(x.external_id)) - } - }; - - to_apply.push(ChronicleOperation::EndActivity(EndActivity { - namespace: namespace.clone(), - id: id.clone(), - time: time.unwrap_or_else(Utc::now).into(), - })); - - if let Some(agent_id) = agent_id { - to_apply.push(ChronicleOperation::was_associated_with( - namespace, - id.clone(), - agent_id, - None, - )); - } - - api.apply_effects_and_submit( - connection, - id, - identity, - to_apply, - applying_new_namespace, - ) - }) - }) - .await? - } - - #[instrument(skip(self))] - async fn use_agent_in_cli_context( - &self, - id: AgentId, - namespace: ExternalId, - ) -> Result { - let api = self.clone(); - tokio::task::spawn_blocking(move || { - let mut connection = api.store.connection()?; - - connection.build_transaction().run(|connection| { - api.store.apply_use_agent(connection, id.external_id_part(), &namespace) - })?; + let mut connection = self.store.connection().await?; + let (namespace, mut to_apply) = self.ensure_namespace(&mut connection, &namespace).await?; + let applying_new_namespace = !to_apply.is_empty(); + + to_apply.push(ChronicleOperation::EndActivity(EndActivity { + namespace: namespace.clone(), + id: id.clone(), + time: time.unwrap_or_else(Utc::now).into(), + })); + + if let Some(agent_id) = agent_id { + to_apply.push(ChronicleOperation::was_associated_with( + namespace, + id.clone(), + agent_id, + None, + )); + } - Ok(ApiResponse::Unit) - }) - .await? + self.apply_effects_and_submit( + &mut connection, + id, + identity, + to_apply, + applying_new_namespace, + ) + .await } } diff --git a/crates/api/src/chronicle_graphql/activity.rs b/crates/api/src/chronicle_graphql/activity.rs index 511ad0f6..90a69008 100644 --- a/crates/api/src/chronicle_graphql/activity.rs +++ b/crates/api/src/chronicle_graphql/activity.rs @@ -1,10 +1,12 @@ -use std::collections::HashMap; - use async_graphql::Context; -use diesel::prelude::*; use chronicle_persistence::queryable::{Activity, Agent, Entity, Namespace}; use common::prov::Role; +use diesel::{ + BoolExpressionMethods, ExpressionMethods, JoinOnDsl, NullableExpressionMethods, + OptionalExtension, QueryDsl, SelectableHelper, +}; +use diesel_async::RunQueryDsl; use crate::chronicle_graphql::DatabaseContext; @@ -15,11 +17,12 @@ pub async fn namespace<'a>( use chronicle_persistence::schema::namespace::{self, dsl}; let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(namespace::table .filter(dsl::id.eq(namespaceid)) - .first::(&mut connection)?) + .first::(&mut connection) + .await?) } pub async fn was_associated_with<'a>( @@ -27,69 +30,38 @@ pub async fn was_associated_with<'a>( ctx: &Context<'a>, ) -> async_graphql::Result, Option, Option)>> { use chronicle_persistence::schema::{agent, association, delegation}; - - #[derive(Queryable)] - struct DelegationAgents { - responsible_id: i32, - delegate: Agent, - role: String, - } + use diesel::alias; let store = ctx.data::()?; - let mut connection = store.connection()?; - - let delegation_entries = delegation::table - .filter(delegation::dsl::activity_id.eq(id)) - .inner_join(agent::table.on(agent::id.eq(delegation::delegate_id))) - .select((delegation::responsible_id, Agent::as_select(), delegation::role)) - .load::(&mut connection)? - .into_iter(); - - let mut agent_reservoir = HashMap::new(); - let mut agent_delegations = HashMap::new(); - - for delegation_entry in delegation_entries { - let delegate_id = delegation_entry.delegate.id; - agent_reservoir.insert(delegate_id, delegation_entry.delegate); - agent_delegations.insert( - delegation_entry.responsible_id, - ( - delegate_id, - if delegation_entry.role.is_empty() { - None - } else { - Some(Role(delegation_entry.role)) - }, - ), - ); - } + let mut connection = store.connection().await?; + + let delegate = alias!(agent as delegate); let res = association::table - .filter(association::dsl::activity_id.eq(id)) - .inner_join(chronicle_persistence::schema::agent::table) - .order(chronicle_persistence::schema::agent::external_id) - .select((Agent::as_select(), association::role)) - .load::<(Agent, Role)>(&mut connection)? + .filter(association::activity_id.eq(id)) + .inner_join(agent::table.on(association::agent_id.eq(agent::id))) + .left_join( + delegation::table.on(delegation::activity_id + .eq(association::activity_id) + .and(delegation::responsible_id.eq(association::agent_id))), + ) + .left_join(delegate.on(delegation::delegate_id.eq(delegate.field(agent::id)))) + .order((agent::external_id, delegate.field(agent::external_id))) + .select(( + Agent::as_select(), + association::role.nullable(), + delegate.fields(agent::all_columns).nullable(), + delegation::role.nullable(), + )) + .load::<(Agent, Option, Option, Option)>(&mut connection) + .await?; + + Ok(res .into_iter() - .map(|(responsible_agent, responsible_role)| { - let responsible_role = - if responsible_role.0.is_empty() { None } else { Some(responsible_role) }; - let (delegate_agent, delegate_role): (Option, Option) = - match agent_delegations.get(&responsible_agent.id) { - Some((delegate_id, optional_role)) => { - let delegate = agent_reservoir.remove(delegate_id).unwrap_or_else(|| { - agent::table.find(delegate_id).first::(&mut connection).unwrap() - }); - let optional_role = optional_role.as_ref().cloned(); - (Some(delegate), optional_role) - }, - None => (None, None), - }; - (responsible_agent, responsible_role, delegate_agent, delegate_role) + .map(|(agent, role, delegate, delegate_role)| { + (agent, role.map(Role::from), delegate, delegate_role.map(Role::from)) }) - .collect(); - - Ok(res) + .collect()) } pub async fn used<'a>(id: i32, ctx: &Context<'a>) -> async_graphql::Result> { @@ -97,14 +69,15 @@ pub async fn used<'a>(id: i32, ctx: &Context<'a>) -> async_graphql::Result()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let res = usage::table .filter(dsl::activity_id.eq(id)) .inner_join(chronicle_persistence::schema::entity::table) .order(chronicle_persistence::schema::entity::external_id) .select(Entity::as_select()) - .load::(&mut connection)?; + .load::(&mut connection) + .await?; Ok(res) } @@ -117,7 +90,7 @@ pub async fn was_informed_by<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let res = wasinformedby::table .filter(dsl::activity_id.eq(id)) @@ -126,7 +99,8 @@ pub async fn was_informed_by<'a>( )) .order(chronicle_persistence::schema::activity::external_id) .select(Activity::as_select()) - .load::(&mut connection)?; + .load::(&mut connection) + .await?; Ok(res) } @@ -136,13 +110,14 @@ pub async fn generated<'a>(id: i32, ctx: &Context<'a>) -> async_graphql::Result< let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let res = generation::table .filter(dsl::activity_id.eq(id)) .inner_join(chronicle_persistence::schema::entity::table) .select(Entity::as_select()) - .load::(&mut connection)?; + .load::(&mut connection) + .await?; Ok(res) } @@ -156,7 +131,7 @@ pub async fn load_attribute<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(activity_attribute::table .filter( @@ -166,6 +141,7 @@ pub async fn load_attribute<'a>( ) .select(activity_attribute::value) .first::(&mut connection) + .await .optional()? .as_deref() .map(serde_json::from_str) diff --git a/crates/api/src/chronicle_graphql/agent.rs b/crates/api/src/chronicle_graphql/agent.rs index 4c459374..a9ee8229 100644 --- a/crates/api/src/chronicle_graphql/agent.rs +++ b/crates/api/src/chronicle_graphql/agent.rs @@ -1,5 +1,9 @@ use async_graphql::Context; -use diesel::prelude::*; +use diesel::{ + BoolExpressionMethods, ExpressionMethods, JoinOnDsl, OptionalExtension, QueryDsl, + SelectableHelper, +}; +use diesel_async::RunQueryDsl; use chronicle_persistence::queryable::{Agent, Entity, Namespace}; use common::prov::Role; @@ -13,11 +17,12 @@ pub async fn namespace<'a>( use chronicle_persistence::schema::namespace::{self, dsl}; let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(namespace::table .filter(dsl::id.eq(namespace_id)) - .first::(&mut connection)?) + .first::(&mut connection) + .await?) } pub async fn acted_on_behalf_of<'a>( @@ -31,14 +36,15 @@ pub async fn acted_on_behalf_of<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(delegation::table .filter(dsl::delegate_id.eq(id)) .inner_join(agentdsl::table.on(dsl::responsible_id.eq(agentdsl::id))) .order(agentdsl::external_id) .select((Agent::as_select(), dsl::role)) - .load::<(Agent, Role)>(&mut connection)? + .load::<(Agent, Role)>(&mut connection) + .await? .into_iter() .map(|(a, r)| (a, if r.0.is_empty() { None } else { Some(r) })) .collect()) @@ -57,14 +63,15 @@ pub async fn attribution<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(attribution::table .filter(dsl::agent_id.eq(id)) .inner_join(entity_dsl::table.on(dsl::entity_id.eq(entity_dsl::id))) .order(entity_dsl::external_id) .select((Entity::as_select(), dsl::role)) - .load::<(Entity, Role)>(&mut connection)? + .load::<(Entity, Role)>(&mut connection) + .await? .into_iter() .map(|(entity, role)| (entity, if role.0.is_empty() { None } else { Some(role) })) .collect()) @@ -79,12 +86,13 @@ pub async fn load_attribute<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(agent_attribute::table .filter(agent_attribute::agent_id.eq(id).and(agent_attribute::typename.eq(external_id))) .select(agent_attribute::value) .first::(&mut connection) + .await .optional()? .as_deref() .map(serde_json::from_str) diff --git a/crates/api/src/chronicle_graphql/entity.rs b/crates/api/src/chronicle_graphql/entity.rs index 44b1b452..f6a34926 100644 --- a/crates/api/src/chronicle_graphql/entity.rs +++ b/crates/api/src/chronicle_graphql/entity.rs @@ -1,5 +1,10 @@ use async_graphql::Context; -use diesel::prelude::*; + +use diesel::{ + BoolExpressionMethods, ExpressionMethods, JoinOnDsl, OptionalExtension, QueryDsl, + SelectableHelper, +}; +use diesel_async::RunQueryDsl; use chronicle_persistence::queryable::{Activity, Agent, Entity, Namespace}; use common::prov::{operations::DerivationType, Role}; @@ -18,13 +23,14 @@ async fn typed_derivation<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let res = derivation::table .filter(dsl::generated_entity_id.eq(id).and(dsl::typ.eq(typ))) .inner_join(entitydsl::table.on(dsl::used_entity_id.eq(entitydsl::id))) .select(Entity::as_select()) - .load::(&mut connection)?; + .load::(&mut connection) + .await?; Ok(res) } @@ -37,11 +43,12 @@ pub async fn namespace<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(namespace::table .filter(dsl::id.eq(namespace_id)) - .first::(&mut connection)?) + .first::(&mut connection) + .await?) } /// Return the agents to which an entity was attributed along with the roles in which it was @@ -53,14 +60,15 @@ pub async fn was_attributed_to<'a>( use chronicle_persistence::schema::{agent, attribution}; let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let res = attribution::table .filter(attribution::dsl::entity_id.eq(id)) .inner_join(agent::table) .order(agent::external_id) .select((Agent::as_select(), attribution::role)) - .load::<(Agent, Role)>(&mut connection)? + .load::<(Agent, Role)>(&mut connection) + .await? .into_iter() .map(|(agent, role)| { let role = if role.0.is_empty() { None } else { Some(role) }; @@ -79,13 +87,14 @@ pub async fn was_generated_by<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let res = generation::table .filter(dsl::generated_entity_id.eq(id)) .inner_join(chronicle_persistence::schema::activity::table) .select(Activity::as_select()) - .load::(&mut connection)?; + .load::(&mut connection) + .await?; Ok(res) } @@ -101,13 +110,14 @@ pub async fn was_derived_from<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let res = derivation::table .filter(dsl::generated_entity_id.eq(id)) .inner_join(entitydsl::table.on(dsl::used_entity_id.eq(entitydsl::id))) .select(Entity::as_select()) - .load::(&mut connection)?; + .load::(&mut connection) + .await?; Ok(res) } @@ -136,7 +146,7 @@ pub async fn load_attribute<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(entity_attribute::table .filter( @@ -146,6 +156,7 @@ pub async fn load_attribute<'a>( ) .select(entity_attribute::value) .first::(&mut connection) + .await .optional()? .as_deref() .map(serde_json::from_str) diff --git a/crates/api/src/chronicle_graphql/mod.rs b/crates/api/src/chronicle_graphql/mod.rs index 04084218..9c4c7743 100644 --- a/crates/api/src/chronicle_graphql/mod.rs +++ b/crates/api/src/chronicle_graphql/mod.rs @@ -17,11 +17,12 @@ use async_graphql_poem::{ GraphQLWebSocket, }; -use diesel::{ - prelude::*, - r2d2::{ConnectionManager, Pool, PooledConnection}, - PgConnection, Queryable, +use diesel::Queryable; +use diesel_async::{ + pooled_connection::deadpool::{Object, Pool}, + AsyncPgConnection, }; + use futures::Stream; use lazy_static::lazy_static; use opentelemetry::global; @@ -462,7 +463,7 @@ impl SecurityConf { pub trait ChronicleApiServer { async fn serve_api( &self, - pool: Pool>, + pool: Pool, api: ApiDispatch, addresses: Vec, security_conf: &SecurityConf, @@ -714,11 +715,7 @@ impl IriEndpoint { prov_type: &str, id: &ID, ns: &ExternalId, - retrieve: impl FnOnce( - PooledConnection>, - &ID, - &ExternalId, - ) -> Result, + retrieve: Result, ) -> poem::Result { match execute_opa_check(&self.opa_executor, &self.claim_parser, claims, |identity| { OpaData::operation( @@ -733,46 +730,38 @@ impl IriEndpoint { }) .await { - Ok(()) => match self.store.connection() { - Ok(connection) => match retrieve(connection, id, ns) { - Ok(data) => match data.to_json().compact().await { - Ok(mut json) => { - use serde_json::Value; - if let Value::Object(mut map) = json { - map.insert( - "@context".to_string(), - Value::String("/context".to_string()), - ); - json = Value::Object(map); - } - Ok(IntoResponse::into_response(poem::web::Json(json))) - }, - Err(error) => { - tracing::error!("JSON failed compaction: {error}"); - Ok(poem::Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body("failed to compact JSON response")) - }, - }, - Err(StoreError::Db(diesel::result::Error::NotFound)) | - Err(StoreError::RecordNotFound) => { - tracing::debug!("not found: {prov_type} {} in {ns}", id.external_id_part()); - Ok(poem::Response::builder() - .status(StatusCode::NOT_FOUND) - .body(format!("the specified {prov_type} does not exist"))) + Ok(()) => match retrieve { + Ok(data) => match data.to_json().compact().await { + Ok(mut json) => { + use serde_json::Value; + if let Value::Object(mut map) = json { + map.insert( + "@context".to_string(), + Value::String("/context".to_string()), + ); + json = Value::Object(map); + } + Ok(IntoResponse::into_response(poem::web::Json(json))) }, Err(error) => { - tracing::error!("failed to retrieve from database: {error}"); + tracing::error!("JSON failed compaction: {error}"); Ok(poem::Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) - .body("failed to fetch from backend storage")) + .body("failed to compact JSON response")) }, }, + Err(StoreError::Db(diesel::result::Error::NotFound)) | + Err(StoreError::RecordNotFound) => { + tracing::debug!("not found: {prov_type} {} in {ns}", id.external_id_part()); + Ok(poem::Response::builder() + .status(StatusCode::NOT_FOUND) + .body(format!("the specified {prov_type} does not exist"))) + }, Err(error) => { - tracing::error!("failed to connect to database: {error}"); + tracing::error!("failed to retrieve from database: {error}"); Ok(poem::Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) - .body("failed to access backend storage")) + .body("failed to fetch from backend storage")) }, }, Err(_) => Ok(poem::Response::builder() @@ -830,22 +819,22 @@ impl IriEndpoint { req: poem::Request, claims: Option<&JwtClaims>, ) -> poem::Result { + let mut conn = self.store.connection().await.map_err(|err| { + poem::Error::from_string(err.to_string(), StatusCode::INTERNAL_SERVER_ERROR) + })?; match self.parse_ns_iri_from_uri_path(req).await? { - Ok((ns, ChronicleIri::Activity(id))) => - self.response_for_query(claims, "activity", &id, &ns, |mut conn, id, ns| { - self.store.prov_model_for_activity_id(&mut conn, id, ns) - }) - .await, - Ok((ns, ChronicleIri::Agent(id))) => - self.response_for_query(claims, "agent", &id, &ns, |mut conn, id, ns| { - self.store.prov_model_for_agent_id(&mut conn, id, ns) - }) - .await, - Ok((ns, ChronicleIri::Entity(id))) => - self.response_for_query(claims, "entity", &id, &ns, |mut conn, id, ns| { - self.store.prov_model_for_entity_id(&mut conn, id, ns) - }) - .await, + Ok((ns, ChronicleIri::Activity(id))) => { + let response = self.store.prov_model_for_activity_id(&mut conn, &id, &ns).await; + self.response_for_query(claims, "activity", &id, &ns, response).await + }, + Ok((ns, ChronicleIri::Agent(id))) => { + let response = self.store.prov_model_for_agent_id(&mut conn, &id, &ns).await; + self.response_for_query(claims, "agent", &id, &ns, response).await + }, + Ok((ns, ChronicleIri::Entity(id))) => { + let response = self.store.prov_model_for_entity_id(&mut conn, &id, &ns).await; + self.response_for_query(claims, "entity", &id, &ns, response).await + }, Ok(_) => Ok(poem::Response::builder() .status(StatusCode::NOT_FOUND) .body("may query only: activity, agent, entity")), @@ -990,16 +979,18 @@ async fn await_shutdown() { #[derive(Clone)] struct DatabaseContext { - pool: Pool>, + pool: Pool, } impl DatabaseContext { - fn new(pool: &Pool>) -> Result { + fn new(pool: &Pool) -> Result { Ok(DatabaseContext { pool: pool.clone() }) } - fn connection(&self) -> Result>, r2d2::Error> { - self.pool.get() + async fn connection( + &self, + ) -> Result, diesel_async::pooled_connection::deadpool::PoolError> { + self.pool.get().await } } @@ -1008,7 +999,7 @@ pub fn construct_schema( mutation: Mutation, subscription: Subscription, claim_parser: Option, - pool: &Pool>, + pool: &Pool, api: &ApiDispatch, opa: ExecutorContext, ) -> Result, StoreError> @@ -1043,7 +1034,7 @@ where { async fn serve_api( &self, - pool: Pool>, + pool: Pool, api: ApiDispatch, addresses: Vec, sec: &SecurityConf, diff --git a/crates/api/src/chronicle_graphql/query.rs b/crates/api/src/chronicle_graphql/query.rs index 0d633854..74d23b61 100644 --- a/crates/api/src/chronicle_graphql/query.rs +++ b/crates/api/src/chronicle_graphql/query.rs @@ -3,7 +3,11 @@ use async_graphql::{ Context, ID, }; use chrono::{DateTime, NaiveDate, TimeZone, Utc}; -use diesel::{debug_query, pg::Pg, prelude::*}; +use diesel::{ + debug_query, pg::Pg, BoolExpressionMethods, ExpressionMethods, JoinOnDsl, + NullableExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, +}; +use diesel_async::RunQueryDsl; use tracing::{debug, instrument}; use chronicle_persistence::{ @@ -40,7 +44,7 @@ pub async fn activity_timeline<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let ns = namespace.unwrap_or_else(|| "default".into()); // Default from and to to the maximum possible time range @@ -122,7 +126,7 @@ pub async fn activity_timeline<'a>( let start = rx.start; let limit = rx.limit; - let rx = rx.load::<(Activity, i64)>(&mut connection)?; + let rx = rx.load::<(Activity, i64)>(&mut connection).await?; Ok::<_, GraphQlError>(project_to_nodes(rx, start, limit)) }) @@ -143,7 +147,7 @@ pub async fn entities_by_type<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let ns = namespace.unwrap_or_else(|| "default".into()); let sql_query = entity::table @@ -163,7 +167,7 @@ pub async fn entities_by_type<'a>( let start = rx.start; let limit = rx.limit; - let rx = rx.load::<(Entity, i64)>(&mut connection)?; + let rx = rx.load::<(Entity, i64)>(&mut connection).await?; Ok::<_, GraphQlError>(project_to_nodes(rx, start, limit)) }) @@ -184,7 +188,7 @@ pub async fn activities_by_type<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let ns = namespace.unwrap_or_else(|| "default".into()); let sql_query = @@ -203,7 +207,7 @@ pub async fn activities_by_type<'a>( let start = rx.start; let limit = rx.limit; - let rx = rx.load::<(Activity, i64)>(&mut connection)?; + let rx = rx.load::<(Activity, i64)>(&mut connection).await?; Ok::<_, GraphQlError>(project_to_nodes(rx, start, limit)) }) @@ -224,7 +228,7 @@ pub async fn agents_by_type<'a>( let store = ctx.data::()?; - let mut connection = store.connection()?; + let mut connection = store.connection().await?; let ns = namespace.unwrap_or_else(|| "default".into()); let sql_query = agent::table @@ -244,7 +248,7 @@ pub async fn agents_by_type<'a>( let start = rx.start; let limit = rx.limit; - let rx = rx.load::<(Agent, i64)>(&mut connection)?; + let rx = rx.load::<(Agent, i64)>(&mut connection).await?; Ok::<_, GraphQlError>(project_to_nodes(rx, start, limit)) }) @@ -264,13 +268,14 @@ pub async fn agent_by_id<'a>( let store = ctx.data::()?; let ns = namespace.unwrap_or_else(|| "default".into()); - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(agent::table .inner_join(nsdsl::namespace) .filter(dsl::external_id.eq(id.external_id_part()).and(nsdsl::external_id.eq(&ns))) .select(Agent::as_select()) .first::(&mut connection) + .await .optional()?) } @@ -287,13 +292,14 @@ pub async fn activity_by_id<'a>( let store = ctx.data::()?; let ns = namespace.unwrap_or_else(|| "default".into()); - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(activity::table .inner_join(nsdsl::namespace) .filter(dsl::external_id.eq(id.external_id_part()).and(nsdsl::external_id.eq(&ns))) .select(Activity::as_select()) .first::(&mut connection) + .await .optional()?) } @@ -309,12 +315,13 @@ pub async fn entity_by_id<'a>( let store = ctx.data::()?; let ns = namespace.unwrap_or_else(|| "default".into()); - let mut connection = store.connection()?; + let mut connection = store.connection().await?; Ok(entity::table .inner_join(nsdsl::namespace) .filter(dsl::external_id.eq(id.external_id_part()).and(nsdsl::external_id.eq(&ns))) .select(Entity::as_select()) .first::(&mut connection) + .await .optional()?) } diff --git a/crates/api/src/commands.rs b/crates/api/src/commands.rs index 34ccc25c..855407e4 100644 --- a/crates/api/src/commands.rs +++ b/crates/api/src/commands.rs @@ -28,10 +28,6 @@ pub enum AgentCommand { namespace: ExternalId, attributes: Attributes, }, - UseInContext { - id: AgentId, - namespace: ExternalId, - }, Delegate { id: AgentId, delegate: AgentId, diff --git a/crates/chronicle-arrow/Cargo.toml b/crates/chronicle-arrow/Cargo.toml index 999ec2e1..e10f7e92 100644 --- a/crates/chronicle-arrow/Cargo.toml +++ b/crates/chronicle-arrow/Cargo.toml @@ -15,6 +15,7 @@ arrow-ipc = { version = "50" } arrow-schema = { version = "50" } chrono = { workspace = true } diesel = { workspace = true } +diesel-async = { workspace = true } futures = { workspace = true } lazy_static = { workspace = true } prost-types = { version = "0.12.3", default-features = false } diff --git a/crates/chronicle-arrow/src/lib.rs b/crates/chronicle-arrow/src/lib.rs index e40ae6fe..dd4a8221 100644 --- a/crates/chronicle-arrow/src/lib.rs +++ b/crates/chronicle-arrow/src/lib.rs @@ -6,17 +6,19 @@ use arrow_flight::{ HandshakeResponse, IpcMessage, PutResult, SchemaAsIpc, SchemaResult, Ticket, }; use arrow_schema::ArrowError; -use diesel::{r2d2::ConnectionManager, PgConnection}; +use diesel_async::{ + pooled_connection::deadpool::{Pool, PoolError}, + AsyncPgConnection, +}; use futures::{ future::join_all, stream::{self, BoxStream}, FutureExt, StreamExt, }; use lazy_static::lazy_static; -use r2d2::Pool; use serde::Serialize; use thiserror::Error; -use tokio::{sync::broadcast, task::spawn_blocking}; +use tokio::sync::broadcast; use tonic::{transport::Server, Request, Response, Status, Streaming}; use tracing::{info, instrument}; @@ -74,6 +76,13 @@ pub enum ChronicleArrowError { #[error("Metadata not found")] MetadataNotFound, + #[error("Pool error: {0}")] + PoolError( + #[from] + #[source] + PoolError, + ), + #[error("API error: {0}")] ApiError( #[from] @@ -87,13 +96,6 @@ pub enum ChronicleArrowError { ParseIriError, ), - #[error("Database connection pool error: {0}")] - PoolError( - #[from] - #[source] - r2d2::Error, - ), - #[error("Diesel error: {0}")] DieselError( #[from] @@ -125,53 +127,47 @@ pub enum ChronicleArrowError { #[instrument(skip(pool, term, domaintype))] pub async fn calculate_count_by_metadata_term( - pool: &Pool>, + pool: &Pool, term: &Term, domaintype: Option, ) -> Result { let pool = pool.clone(); match term { Term::Entity => - spawn_blocking(move || { - entity_count_by_type( - &pool, - domaintype.map(|x| x.to_string()).iter().map(|s| s.as_str()).collect(), - ) - }) + entity_count_by_type( + &pool, + domaintype.map(|x| x.to_string()).iter().map(|s| s.as_str()).collect(), + ) .await, Term::Agent => - spawn_blocking(move || { - agent_count_by_type( - &pool, - domaintype.map(|x| x.to_string()).iter().map(|s| s.as_str()).collect(), - ) - }) + agent_count_by_type( + &pool, + domaintype.map(|x| x.to_string()).iter().map(|s| s.as_str()).collect(), + ) .await, Term::Activity => - spawn_blocking(move || { - activity_count_by_type( - &pool, - domaintype.map(|x| x.to_string()).iter().map(|s| s.as_str()).collect(), - ) - }) + activity_count_by_type( + &pool, + domaintype.map(|x| x.to_string()).iter().map(|s| s.as_str()).collect(), + ) .await, - _ => Ok(Ok(0)), + _ => Ok(0), } .map_err(|e| Status::from_error(e.into())) - .and_then(|res| res.map_err(|e| Status::from_error(e.into()))) } async fn create_flight_info_for_type( - pool: Arc>>, + pool: Arc>, domain_items: Vec, term: Term, record_batch_size: usize, ) -> BoxStream<'static, Result> { + let pool = Arc::clone(&pool); stream::iter(domain_items.into_iter().map(|item| Ok::<_, tonic::Status>(item))) .then(move |item| { - let pool = pool.clone(); + let pool = Arc::clone(&pool); async move { - let item = item?; // Handle the Result from the iterator + let item = item?; let descriptor_path = vec![term.to_string(), item.as_type_name()]; let metadata = get_domain_type_meta_from_cache(&descriptor_path).ok_or_else(|| { @@ -185,21 +181,24 @@ async fn create_flight_info_for_type( ) .await?; - let tickets = (0..count) - .step_by(record_batch_size as _) - .map(|start| { - let end = std::cmp::min(start as usize + record_batch_size, count as usize); - - let ticket_metadata = ChronicleTicket::new( - term, - metadata.typ.as_ref().map(|x| x.as_domain_type_id()), - start as _, - (end - start as usize) as _, - ); + let tickets = (0..count).step_by(record_batch_size as _).map(|start| { + let end = std::cmp::min(start as usize + record_batch_size, count as usize); + + let ticket_metadata = ChronicleTicket::new( + term, + metadata.typ.as_ref().map(|x| x.as_domain_type_id()), + start as _, + (end - start as usize) as _, + ); + async move { Ticket::try_from(ticket_metadata) .map_err(|e| Status::from_error(Box::new(ChronicleArrowError::from(e)))) - }) - .collect::, _>>()?; + } + }); + + let tickets: Result, _> = + futures::future::join_all(tickets).await.into_iter().collect(); + let tickets = tickets?; let mut flight_info = FlightInfo::new(); @@ -221,7 +220,7 @@ async fn create_flight_info_for_type( #[derive(Clone)] pub struct FlightServiceImpl { domain: common::domain::ChronicleDomainDef, - pool: r2d2::Pool>, + pool: Pool, api: ApiDispatch, record_batch_size: usize, security: EndpointSecurityConfiguration, @@ -230,7 +229,7 @@ pub struct FlightServiceImpl { impl FlightServiceImpl { pub fn new( domain: &common::domain::ChronicleDomainDef, - pool: &r2d2::Pool>, + pool: &Pool, api: &ApiDispatch, security: EndpointSecurityConfiguration, record_batch_size: usize, @@ -484,19 +483,15 @@ impl FlightService for FlightServiceImpl { let terms_result = match ticket.term { Term::Entity => { - let pool = self.pool.clone(); let meta_clone = meta.clone(); - let result = tokio::task::spawn_blocking(move || { - load_entities_by_type( - &pool, - &ticket.typ, - &meta_clone.attributes, - ticket.start, - ticket.count, - ) - }) + let result = load_entities_by_type( + &self.pool, + &ticket.typ, + &meta_clone.attributes, + ticket.start, + ticket.count, + ) .await - .map_err(|e| Status::from_error(Box::new(ChronicleArrowError::from(e))))? .map_err(|e| Status::from_error(Box::new(e)))?; let (entities, _returned_records, _total_records) = result; @@ -507,12 +502,10 @@ impl FlightService for FlightServiceImpl { }, Term::Activity => { let pool = self.pool.clone(); - let result = tokio::task::spawn_blocking(move || { + let result = load_activities_by_type(&pool, &ticket.typ, ticket.start, ticket.count) - }) - .await - .map_err(|e| Status::from_error(Box::new(ChronicleArrowError::from(e))))? - .map_err(|e| Status::from_error(Box::new(e)))?; + .await + .map_err(|e| Status::from_error(Box::new(e)))?; let (activities, _returned_records, _total_records) = result; @@ -522,12 +515,9 @@ impl FlightService for FlightServiceImpl { }, Term::Agent => { let pool = self.pool.clone(); - let result = tokio::task::spawn_blocking(move || { - load_agents_by_type(&pool, &ticket.typ, ticket.start, ticket.count) - }) - .await - .map_err(|e| Status::from_error(Box::new(ChronicleArrowError::from(e))))? - .map_err(|e| Status::from_error(Box::new(e)))?; + let result = load_agents_by_type(&pool, &ticket.typ, ticket.start, ticket.count) + .await + .map_err(|e| Status::from_error(Box::new(e)))?; let (agents, _returned_records, _total_records) = result; @@ -644,7 +634,7 @@ pub async fn await_shutdown() { #[instrument(skip(pool, api, security))] pub async fn run_flight_service( domain: &common::domain::ChronicleDomainDef, - pool: &Pool>, + pool: &Pool, api: &ApiDispatch, security: EndpointSecurityConfiguration, addrs: &Vec, diff --git a/crates/chronicle-arrow/src/query/activity.rs b/crates/chronicle-arrow/src/query/activity.rs index 2edf5a95..b8a2a070 100644 --- a/crates/chronicle-arrow/src/query/activity.rs +++ b/crates/chronicle-arrow/src/query/activity.rs @@ -6,11 +6,9 @@ use arrow_array::{ }; use arrow_schema::{DataType, Field}; use chrono::{DateTime, Utc}; -use diesel::{ - pg::PgConnection, - prelude::*, - r2d2::{ConnectionManager, Pool}, -}; + +use diesel::{BelongingToDsl, ExpressionMethods, JoinOnDsl, QueryDsl, SelectableHelper}; +use diesel_async::{pooled_connection::deadpool::Pool, AsyncPgConnection, RunQueryDsl}; use uuid::Uuid; use chronicle_persistence::{ @@ -31,15 +29,16 @@ use crate::{ChronicleArrowError, DomainTypeMeta}; use super::vec_vec_string_to_list_array; #[tracing::instrument(skip(pool))] -pub fn activity_count_by_type( - pool: &Pool>, +pub async fn activity_count_by_type( + pool: &Pool, typ: Vec<&str>, ) -> Result { - let mut connection = pool.get()?; + let mut connection = pool.get().await?; let count = activity::table .filter(activity::domaintype.eq_any(typ)) .count() - .get_result(&mut connection)?; + .get_result(&mut connection) + .await?; Ok(count) } @@ -283,31 +282,35 @@ fn associations_to_list_array( Ok(builder.finish()) } -pub fn load_activities_by_type( - pool: &Pool>, +pub async fn load_activities_by_type( + pool: &Pool, typ: &Option, position: u64, max_records: u64, ) -> Result<(impl Iterator, u64, u64), ChronicleArrowError> { - let mut connection = pool.get().map_err(ChronicleArrowError::PoolError)?; + let mut connection = pool.get().await.map_err(ChronicleArrowError::PoolError)?; let activities_and_namespaces: Vec<(Activity, Namespace)> = match typ { - Some(typ_value) => activity::table - .inner_join(namespace::table.on(activity::namespace_id.eq(namespace::id))) - .filter(activity::domaintype.eq(typ_value.external_id_part())) - .order(activity::id) - .select((Activity::as_select(), Namespace::as_select())) - .offset(position as i64) - .limit(max_records as i64) - .load(&mut connection)?, - None => activity::table - .inner_join(namespace::table.on(activity::namespace_id.eq(namespace::id))) - .filter(activity::domaintype.is_null()) - .order(activity::id) - .select((Activity::as_select(), Namespace::as_select())) - .offset(position as i64) - .limit(max_records as i64) - .load(&mut connection)?, + Some(typ_value) => + activity::table + .inner_join(namespace::table.on(activity::namespace_id.eq(namespace::id))) + .filter(activity::domaintype.eq(typ_value.external_id_part())) + .order(activity::id) + .select((Activity::as_select(), Namespace::as_select())) + .offset(position as i64) + .limit(max_records as i64) + .load(&mut connection) + .await?, + None => + activity::table + .inner_join(namespace::table.on(activity::namespace_id.eq(namespace::id))) + .filter(activity::domaintype.is_null()) + .order(activity::id) + .select((Activity::as_select(), Namespace::as_select())) + .offset(position as i64) + .limit(max_records as i64) + .load(&mut connection) + .await?, }; let (activities, namespaces): (Vec, Vec) = @@ -317,7 +320,8 @@ pub fn load_activities_by_type( WasInformedBy::belonging_to(&activities) .inner_join(activity::table.on(wasinformedby::informing_activity_id.eq(activity::id))) .select((wasinformedby::activity_id, activity::external_id)) - .load::<(i32, String)>(&mut connection)? + .load::<(i32, String)>(&mut connection) + .await? .into_iter() .fold(HashMap::new(), |mut acc: HashMap>, (id, external_id)| { acc.entry(id).or_default().push(external_id); @@ -327,7 +331,8 @@ pub fn load_activities_by_type( let mut used_map: HashMap> = Usage::belonging_to(&activities) .inner_join(entity::table.on(usage::entity_id.eq(entity::id))) .select((usage::activity_id, entity::external_id)) - .load::<(i32, String)>(&mut connection)? + .load::<(i32, String)>(&mut connection) + .await? .into_iter() .fold(HashMap::new(), |mut acc: HashMap>, (id, external_id)| { acc.entry(id).or_default().push(external_id); @@ -337,7 +342,8 @@ pub fn load_activities_by_type( let mut generated_map: HashMap> = Generation::belonging_to(&activities) .inner_join(entity::table.on(generation::generated_entity_id.eq(entity::id))) .select((generation::activity_id, entity::external_id)) - .load::<(i32, String)>(&mut connection)? + .load::<(i32, String)>(&mut connection) + .await? .into_iter() .fold(HashMap::new(), |mut acc: HashMap>, (id, external_id)| { acc.entry(id).or_default().push(external_id); @@ -348,7 +354,8 @@ pub fn load_activities_by_type( Association::belonging_to(&activities) .inner_join(agent::table.on(association::agent_id.eq(agent::id))) .select((association::activity_id, (agent::id, agent::external_id, association::role))) - .load::<(i32, (i32, String, String))>(&mut connection)? + .load::<(i32, (i32, String, String))>(&mut connection) + .await? .into_iter() .fold( HashMap::new(), @@ -368,7 +375,8 @@ pub fn load_activities_by_type( delegation::activity_id, (delegation::responsible_id, agent::external_id, delegation::role), )) - .load::<(i32, (i32, String, String))>(&mut connection)? + .load::<(i32, (i32, String, String))>(&mut connection) + .await? .into_iter() .fold( HashMap::new(), diff --git a/crates/chronicle-arrow/src/query/agent.rs b/crates/chronicle-arrow/src/query/agent.rs index 4a3ab563..ec05151d 100644 --- a/crates/chronicle-arrow/src/query/agent.rs +++ b/crates/chronicle-arrow/src/query/agent.rs @@ -5,11 +5,8 @@ use arrow_array::{Array, BooleanArray, Int64Array, ListArray, RecordBatch, Strin use arrow_buffer::{Buffer, ToByteSlice}; use arrow_data::ArrayData; use arrow_schema::{DataType, Field}; -use diesel::{ - pg::PgConnection, - prelude::*, - r2d2::{ConnectionManager, Pool}, -}; +use diesel::prelude::*; +use diesel_async::{pooled_connection::deadpool::Pool, AsyncPgConnection, RunQueryDsl}; use uuid::Uuid; use chronicle_persistence::{ @@ -28,15 +25,16 @@ use crate::{ }; #[tracing::instrument(skip(pool))] -pub fn agent_count_by_type( - pool: &Pool>, +pub async fn agent_count_by_type( + pool: &Pool, typ: Vec<&str>, ) -> Result { - let mut connection = pool.get()?; + let mut connection = pool.get().await?; let count = agent::table .filter(agent::domaintype.eq_any(typ)) .count() - .get_result(&mut connection)?; + .get_result(&mut connection) + .await?; Ok(count) } @@ -287,31 +285,35 @@ fn agent_attributions_to_list_array( } #[tracing::instrument(skip(pool))] -pub fn load_agents_by_type( - pool: &Pool>, +pub async fn load_agents_by_type( + pool: &Pool, typ: &Option, position: u64, max_records: u64, ) -> Result<(impl Iterator, u64, u64), ChronicleArrowError> { - let mut connection = pool.get().map_err(ChronicleArrowError::PoolError)?; + let mut connection = pool.get().await.map_err(ChronicleArrowError::PoolError)?; let agents_and_namespaces: Vec<(Agent, Namespace)> = match typ { - Some(typ_value) => agent::table - .inner_join(namespace::table.on(agent::namespace_id.eq(namespace::id))) - .filter(agent::domaintype.eq(typ_value.external_id_part())) - .order(agent::id) - .select((Agent::as_select(), Namespace::as_select())) - .offset(position as i64) - .limit(max_records as i64) - .load(&mut connection)?, - None => agent::table - .inner_join(namespace::table.on(agent::namespace_id.eq(namespace::id))) - .filter(agent::domaintype.is_null()) - .order(agent::id) - .select((Agent::as_select(), Namespace::as_select())) - .offset(position as i64) - .limit(max_records as i64) - .load(&mut connection)?, + Some(typ_value) => + agent::table + .inner_join(namespace::table.on(agent::namespace_id.eq(namespace::id))) + .filter(agent::domaintype.eq(typ_value.external_id_part())) + .order(agent::id) + .select((Agent::as_select(), Namespace::as_select())) + .offset(position as i64) + .limit(max_records as i64) + .load(&mut connection) + .await?, + None => + agent::table + .inner_join(namespace::table.on(agent::namespace_id.eq(namespace::id))) + .filter(agent::domaintype.is_null()) + .order(agent::id) + .select((Agent::as_select(), Namespace::as_select())) + .offset(position as i64) + .limit(max_records as i64) + .load(&mut connection) + .await?, }; let total_records = agents_and_namespaces.len() as u64; @@ -323,7 +325,8 @@ pub fn load_agents_by_type( Attribution::belonging_to(&agents) .inner_join(entity::table.on(attribution::entity_id.eq(entity::id))) .select((attribution::agent_id, attribution::role, entity::external_id)) - .load::<(i32, Role, String)>(&mut connection)? + .load::<(i32, Role, String)>(&mut connection) + .await? .into_iter() .fold( HashMap::new(), @@ -346,7 +349,8 @@ pub fn load_agents_by_type( activity::external_id, agent::external_id, )) - .load::<(i32, Role, String, String)>(&mut connection)? + .load::<(i32, Role, String, String)>(&mut connection) + .await? .into_iter() .fold( HashMap::new(), diff --git a/crates/chronicle-arrow/src/query/entity.rs b/crates/chronicle-arrow/src/query/entity.rs index 49d962a4..5e4a56f6 100644 --- a/crates/chronicle-arrow/src/query/entity.rs +++ b/crates/chronicle-arrow/src/query/entity.rs @@ -5,11 +5,8 @@ use arrow_array::{Array, BooleanArray, Int64Array, ListArray, RecordBatch, Strin use arrow_buffer::{Buffer, ToByteSlice}; use arrow_data::ArrayData; use arrow_schema::{DataType, Field}; -use diesel::{ - pg::PgConnection, - prelude::*, - r2d2::{ConnectionManager, Pool}, -}; +use diesel::prelude::*; +use diesel_async::{pooled_connection::deadpool::Pool, AsyncPgConnection, RunQueryDsl}; use uuid::Uuid; use chronicle_persistence::{ @@ -34,16 +31,17 @@ use super::vec_vec_string_to_list_array; // Returns a list of all indexed domain types from entities, activities and agents , note that these // may no longer be present in the domain definition #[tracing::instrument(skip(pool))] -pub fn term_types( - pool: &Pool>, +pub async fn term_types( + pool: Pool, ) -> Result, ChronicleArrowError> { - let mut connection = pool.get()?; + let mut connection = pool.get().await?; let types = entity::table .select(entity::domaintype) .distinct() .union(agent::table.select(agent::domaintype).distinct()) .union(activity::table.select(activity::domaintype).distinct()) - .load::>(&mut connection)?; + .load::>(&mut connection) + .await?; let mut unique_types = types.into_iter().collect::>(); unique_types.sort(); @@ -55,15 +53,16 @@ pub fn term_types( .collect()) } -pub fn entity_count_by_type( - pool: &Pool>, +pub async fn entity_count_by_type( + pool: &Pool, typ: Vec<&str>, ) -> Result { - let mut connection = pool.get()?; + let mut connection = pool.get().await?; let count = entity::table .filter(entity::domaintype.eq_any(typ)) .count() - .get_result(&mut connection)?; + .get_result(&mut connection) + .await?; Ok(count) } @@ -335,14 +334,14 @@ fn attributions_to_list_array( // Returns a tuple of an iterator over entities of the specified domain types and their relations, // the number of returned records and the total number of records #[tracing::instrument(skip(pool))] -pub fn load_entities_by_type( - pool: &Pool>, +pub async fn load_entities_by_type( + pool: &Pool, typ: &Option, attributes: &Vec<(String, PrimitiveType)>, position: u64, max_records: u64, ) -> Result<(impl Iterator, u64, u64), ChronicleArrowError> { - let mut connection = pool.get()?; + let mut connection = pool.get().await?; let mut entities_and_references = Vec::new(); @@ -354,7 +353,8 @@ pub fn load_entities_by_type( .select((Entity::as_select(), Namespace::as_select())) .offset(position as i64) .limit(max_records as i64) - .load::<(Entity, Namespace)>(&mut connection)? + .load::<(Entity, Namespace)>(&mut connection) + .await? } else { entity::table .inner_join(namespace::table.on(entity::namespace_id.eq(namespace::id))) @@ -363,7 +363,8 @@ pub fn load_entities_by_type( .select((Entity::as_select(), Namespace::as_select())) .offset(position as i64) .limit(max_records as i64) - .load::<(Entity, Namespace)>(&mut connection)? + .load::<(Entity, Namespace)>(&mut connection) + .await? }; let (entities, namespaces): (Vec, Vec) = @@ -376,7 +377,8 @@ pub fn load_entities_by_type( .filter(entity_attribute::entity_id.eq_any(&entity_ids)) .filter(entity_attribute::typename.eq_any(&attribute_names)) .select((entity_attribute::entity_id, entity_attribute::typename, entity_attribute::value)) - .load::<(i32, String, String)>(&mut connection)? + .load::<(i32, String, String)>(&mut connection) + .await? .into_iter() .map(|(entity_id, typename, value)| { let parsed_value: serde_json::Value = serde_json::from_str(&value).unwrap_or_default(); @@ -395,7 +397,8 @@ pub fn load_entities_by_type( let mut generation_map: HashMap> = Generation::belonging_to(&entities) .inner_join(activity::table) .select((generation::generated_entity_id, activity::external_id)) - .load::<(i32, String)>(&mut connection)? + .load::<(i32, String)>(&mut connection) + .await? .into_iter() .fold(HashMap::new(), |mut acc: HashMap>, (id, external_id)| { acc.entry(id).or_default().push(external_id); @@ -405,7 +408,8 @@ pub fn load_entities_by_type( let mut attribution_map: HashMap> = Attribution::belonging_to(&entities) .inner_join(agent::table) .select((attribution::agent_id, agent::external_id, attribution::role.nullable())) - .load::<(i32, String, Option)>(&mut connection)? + .load::<(i32, String, Option)>(&mut connection) + .await? .into_iter() .fold(HashMap::new(), |mut acc: HashMap>, (id, external_id, role)| { acc.entry(id) @@ -424,7 +428,8 @@ pub fn load_entities_by_type( entity::external_id, derivation::typ, )) - .load::<(i32, String, String, i32)>(&mut connection)? + .load::<(i32, String, String, i32)>(&mut connection) + .await? .into_iter() .map(|(entity_id, activity_external_id, entity_external_id, derivation_type)| { DerivationType::try_from(derivation_type) diff --git a/crates/chronicle-domain-test/src/test.rs b/crates/chronicle-domain-test/src/test.rs index 99545b3c..808d9787 100644 --- a/crates/chronicle-domain-test/src/test.rs +++ b/crates/chronicle-domain-test/src/test.rs @@ -137,10 +137,9 @@ mod test { let liveness_check_interval = None; - let dispatch = Api::new( + let dispatch = Api::create_dispatch( pool.clone(), embed_substrate, - SameUuid, secrets, vec![], None, @@ -620,7 +619,7 @@ mod test { .unwrap() .chars() .count(), - 32 + 64 ); "[txId]" }), diff --git a/crates/chronicle-persistence/Cargo.toml b/crates/chronicle-persistence/Cargo.toml index 031170bc..cf107334 100644 --- a/crates/chronicle-persistence/Cargo.toml +++ b/crates/chronicle-persistence/Cargo.toml @@ -16,6 +16,7 @@ async-trait = { workspace = true } chrono = { workspace = true } derivative = { workspace = true } diesel = { workspace = true } +diesel-async = { workspace = true } diesel_migrations = { workspace = true } hex = { workspace = true } r2d2 = { version = "^0.8.1" } @@ -23,6 +24,7 @@ serde_json = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } +futures = { workspace = true } #local dependencies common = { path = "../common", features = ["diesel-bindings", "std"] } diff --git a/crates/chronicle-persistence/src/cursor.rs b/crates/chronicle-persistence/src/cursor.rs index e994eb3d..6b49f16c 100644 --- a/crates/chronicle-persistence/src/cursor.rs +++ b/crates/chronicle-persistence/src/cursor.rs @@ -1,16 +1,12 @@ use diesel::{ pg::Pg, - prelude::*, - query_builder::*, - r2d2::{ConnectionManager, PooledConnection}, + query_builder::{AstPass, Query, QueryFragment}, sql_types::BigInt, + QueryResult, }; - -type Conn = PooledConnection>; - const DEFAULT_PAGE_SIZE: i32 = 10; -#[derive(QueryId)] +#[derive(diesel::query_builder::QueryId)] pub struct CursorPosition { query: T, pub start: i64, @@ -66,5 +62,3 @@ where impl Query for CursorPosition { type SqlType = (T::SqlType, BigInt); } - -impl RunQueryDsl for CursorPosition {} diff --git a/crates/chronicle-persistence/src/database.rs b/crates/chronicle-persistence/src/database.rs index 5f700cac..68266231 100644 --- a/crates/chronicle-persistence/src/database.rs +++ b/crates/chronicle-persistence/src/database.rs @@ -1,17 +1,16 @@ -use diesel::{r2d2::ConnectionManager, PgConnection}; - -use diesel::r2d2::Pool; use std::{fmt::Display, time::Duration}; +use diesel_async::{pooled_connection::deadpool::Pool, AsyncPgConnection}; + #[async_trait::async_trait] pub trait DatabaseConnector { - async fn try_connect(&self) -> Result<(X, Pool>), E>; + async fn try_connect(&self) -> Result<(X, Pool), E>; fn should_retry(&self, error: &E) -> bool; } pub async fn get_connection_with_retry( connector: impl DatabaseConnector, -) -> Result<(X, Pool>), E> { +) -> Result<(X, Pool), E> { let mut i = 1; let mut j = 1; loop { diff --git a/crates/chronicle-persistence/src/lib.rs b/crates/chronicle-persistence/src/lib.rs index 15345d1c..5579d298 100644 --- a/crates/chronicle-persistence/src/lib.rs +++ b/crates/chronicle-persistence/src/lib.rs @@ -11,10 +11,10 @@ use common::{ }, }; -use diesel::{ - prelude::*, - r2d2::{ConnectionManager, Pool, PooledConnection}, - PgConnection, +use diesel::prelude::*; +use diesel_async::{ + pooled_connection::{deadpool::Pool, PoolError}, + AsyncPgConnection, RunQueryDsl, }; use diesel_migrations::{embed_migrations, EmbeddedMigrations}; use protocol_substrate_chronicle::protocol::BlockId; @@ -47,6 +47,13 @@ pub enum StoreError { diesel::ConnectionError, ), + #[error("Diesel async pool error: {0}")] + DieselAsyncPool( + #[from] + #[source] + PoolError, + ), + #[error("Database migration failed: {0}")] DbMigration( #[from] @@ -58,7 +65,14 @@ pub enum StoreError { DbPool( #[from] #[source] - r2d2::Error, + diesel_async::pooled_connection::deadpool::PoolError, + ), + + #[error("Db pool builder: {0}")] + DbPoolBuilder( + #[from] + #[source] + diesel_async::pooled_connection::deadpool::BuildError, ), #[error("Infallible")] @@ -116,7 +130,7 @@ pub struct ConnectionOptions { #[derive(Clone)] pub struct Store { - pool: Pool>, + pool: Pool, } type Generations = Vec; type Usages = Vec; @@ -125,86 +139,100 @@ type Associations = Vec<(String, String)>; impl Store { #[instrument(skip(self))] - pub fn namespace_binding(&self, external_id: &str, uuid: Uuid) -> Result<(), StoreError> { + pub async fn namespace_binding(&self, external_id: &str, uuid: Uuid) -> Result<(), StoreError> { use schema::namespace::dsl; let uuid = uuid.to_string(); - self.connection()?.build_transaction().run(|conn| { - diesel::insert_into(dsl::namespace) - .values((dsl::external_id.eq(external_id), dsl::uuid.eq(&uuid))) - .on_conflict(dsl::external_id) - .do_update() - .set(dsl::uuid.eq(&uuid)) - .execute(conn) - })?; + self.connection() + .await? + .build_transaction() + .run(|conn| { + Box::pin( + diesel::insert_into(dsl::namespace) + .values((dsl::external_id.eq(external_id), dsl::uuid.eq(&uuid))) + .on_conflict(dsl::external_id) + .do_update() + .set(dsl::uuid.eq(&uuid)) + .execute(conn), + ) + }) + .await?; Ok(()) } /// Fetch the activity record for the IRI - fn activity_by_activity_external_id_and_namespace( + pub async fn activity_by_activity_external_id_and_namespace( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, external_id: &ExternalId, namespace_id: &NamespaceId, ) -> Result { - let (_namespaceid, nsid) = - self.namespace_by_external_id(connection, namespace_id.external_id_part())?; + let (_namespaceid, nsid) = self + .namespace_by_external_id(connection, namespace_id.external_id_part()) + .await?; use schema::activity::dsl; Ok(schema::activity::table .filter(dsl::external_id.eq(external_id).and(dsl::namespace_id.eq(nsid))) - .first::(connection)?) + .first::(connection) + .await?) } /// Fetch the entity record for the IRI - fn entity_by_entity_external_id_and_namespace( + pub async fn entity_by_entity_external_id_and_namespace( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, external_id: &ExternalId, namespace_id: &NamespaceId, ) -> Result { - let (_, ns_id) = - self.namespace_by_external_id(connection, namespace_id.external_id_part())?; + let (_, ns_id) = self + .namespace_by_external_id(connection, namespace_id.external_id_part()) + .await?; use schema::entity::dsl; Ok(schema::entity::table .filter(dsl::external_id.eq(external_id).and(dsl::namespace_id.eq(ns_id))) - .first::(connection)?) + .first::(connection) + .await?) } /// Fetch the agent record for the IRI - pub fn agent_by_agent_external_id_and_namespace( + pub async fn agent_by_agent_external_id_and_namespace( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, external_id: &ExternalId, namespace_id: &NamespaceId, ) -> Result { - let (_namespaceid, nsid) = - self.namespace_by_external_id(connection, namespace_id.external_id_part())?; + let (_namespaceid, nsid) = self + .namespace_by_external_id(connection, namespace_id.external_id_part()) + .await?; use schema::agent::dsl; Ok(schema::agent::table .filter(dsl::external_id.eq(external_id).and(dsl::namespace_id.eq(nsid))) - .first::(connection)?) + .first::(connection) + .await?) } /// Apply an activity to persistent storage, name + namespace are a key, so we update times + /// domaintype on conflict #[instrument(level = "trace", skip(self, connection), ret(Debug))] - fn apply_activity( + async fn apply_activity( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, Activity { ref external_id, namespace_id, started, ended, domaintype_id, attributes, .. }: &Activity, ) -> Result<(), StoreError> { use schema::activity as dsl; - let (_, nsid) = - self.namespace_by_external_id(connection, namespace_id.external_id_part())?; + let (_, nsid) = self + .namespace_by_external_id(connection, namespace_id.external_id_part()) + .await?; let existing = self .activity_by_activity_external_id_and_namespace(connection, external_id, namespace_id) + .await .ok(); let resolved_domain_type = @@ -234,13 +262,12 @@ impl Store { dsl::started.eq(resolved_started), dsl::ended.eq(resolved_ended), )) - .execute(connection)?; + .execute(connection) + .await?; - let query::Activity { id, .. } = self.activity_by_activity_external_id_and_namespace( - connection, - external_id, - namespace_id, - )?; + let query::Activity { id, .. } = self + .activity_by_activity_external_id_and_namespace(connection, external_id, namespace_id) + .await?; diesel::insert_into(schema::activity_attribute::table) .values( @@ -254,7 +281,8 @@ impl Store { .collect::>(), ) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } @@ -263,17 +291,19 @@ impl Store { /// publickey + domaintype on conflict current is a special case, only relevant to local CLI /// context. A possibly improved design would be to store this in another table given its scope #[instrument(level = "trace", skip(self, connection), ret(Debug))] - fn apply_agent( + async fn apply_agent( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, Agent { ref external_id, namespaceid, domaintypeid, attributes, .. }: &Agent, ) -> Result<(), StoreError> { use schema::agent::dsl; - let (_, nsid) = - self.namespace_by_external_id(connection, namespaceid.external_id_part())?; + let (_, nsid) = self + .namespace_by_external_id(connection, namespaceid.external_id_part()) + .await?; let existing = self .agent_by_agent_external_id_and_namespace(connection, external_id, namespaceid) + .await .ok(); let resolved_domain_type = @@ -291,10 +321,12 @@ impl Store { .on_conflict((dsl::namespace_id, dsl::external_id)) .do_update() .set(dsl::domaintype.eq(resolved_domain_type)) - .execute(connection)?; + .execute(connection) + .await?; - let query::Agent { id, .. } = - self.agent_by_agent_external_id_and_namespace(connection, external_id, namespaceid)?; + let query::Agent { id, .. } = self + .agent_by_agent_external_id_and_namespace(connection, external_id, namespaceid) + .await?; diesel::insert_into(schema::agent_attribute::table) .values( @@ -308,23 +340,26 @@ impl Store { .collect::>(), ) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } #[instrument(level = "trace", skip(self, connection), ret(Debug))] - fn apply_entity( + async fn apply_entity( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, Entity { namespace_id, external_id, domaintypeid, attributes, .. }: &Entity, ) -> Result<(), StoreError> { use schema::entity::dsl; - let (_, nsid) = - self.namespace_by_external_id(connection, namespace_id.external_id_part())?; + let (_, nsid) = self + .namespace_by_external_id(connection, namespace_id.external_id_part()) + .await?; let existing = self .entity_by_entity_external_id_and_namespace(connection, external_id, namespace_id) + .await .ok(); let resolved_domain_type = @@ -341,10 +376,12 @@ impl Store { .on_conflict((dsl::namespace_id, dsl::external_id)) .do_update() .set(dsl::domaintype.eq(resolved_domain_type)) - .execute(connection)?; + .execute(connection) + .await?; - let query::Entity { id, .. } = - self.entity_by_entity_external_id_and_namespace(connection, external_id, namespace_id)?; + let query::Entity { id, .. } = self + .entity_by_entity_external_id_and_namespace(connection, external_id, namespace_id) + .await?; diesel::insert_into(schema::entity_attribute::table) .values( @@ -358,38 +395,39 @@ impl Store { .collect::>(), ) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } - fn apply_model( + async fn apply_model( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, model: &ProvModel, ) -> Result<(), StoreError> { for (_, ns) in model.namespaces.iter() { - self.apply_namespace(connection, ns)? + self.apply_namespace(connection, ns).await? } for (_, agent) in model.agents.iter() { - self.apply_agent(connection, agent)? + self.apply_agent(connection, agent).await? } for (_, activity) in model.activities.iter() { - self.apply_activity(connection, activity)? + self.apply_activity(connection, activity).await? } for (_, entity) in model.entities.iter() { - self.apply_entity(connection, entity)? + self.apply_entity(connection, entity).await? } for ((namespaceid, _), association) in model.association.iter() { for association in association.iter() { - self.apply_was_associated_with(connection, namespaceid, association)?; + self.apply_was_associated_with(connection, namespaceid, association).await?; } } for ((namespaceid, _), usage) in model.usage.iter() { for usage in usage.iter() { - self.apply_used(connection, namespaceid, usage)?; + self.apply_used(connection, namespaceid, usage).await?; } } @@ -400,31 +438,32 @@ impl Store { namespaceid, activity_id, informing_activity_id, - )?; + ) + .await?; } } for ((namespaceid, _), generation) in model.generation.iter() { for generation in generation.iter() { - self.apply_was_generated_by(connection, namespaceid, generation)?; + self.apply_was_generated_by(connection, namespaceid, generation).await?; } } for ((namespaceid, _), derivation) in model.derivation.iter() { for derivation in derivation.iter() { - self.apply_derivation(connection, namespaceid, derivation)?; + self.apply_derivation(connection, namespaceid, derivation).await?; } } for ((namespaceid, _), delegation) in model.delegation.iter() { for delegation in delegation.iter() { - self.apply_delegation(connection, namespaceid, delegation)?; + self.apply_delegation(connection, namespaceid, delegation).await?; } } for ((namespace_id, _), attribution) in model.attribution.iter() { for attribution in attribution.iter() { - self.apply_was_attributed_to(connection, namespace_id, attribution)?; + self.apply_was_attributed_to(connection, namespace_id, attribution).await?; } } @@ -432,46 +471,53 @@ impl Store { } #[instrument(level = "trace", skip(self, connection), ret(Debug))] - fn apply_namespace( + async fn apply_namespace( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, Namespace { ref external_id, ref uuid, .. }: &Namespace, ) -> Result<(), StoreError> { use schema::namespace::dsl; diesel::insert_into(schema::namespace::table) .values((dsl::external_id.eq(external_id), dsl::uuid.eq(hex::encode(uuid)))) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } - pub fn apply_prov(&self, prov: &ProvModel) -> Result<(), StoreError> { - self.connection()? + pub async fn apply_prov(&self, prov: &ProvModel) -> Result<(), StoreError> { + self.connection() + .await? .build_transaction() - .run(|connection| self.apply_model(connection, prov))?; + .run(|connection| Box::pin(self.apply_model(connection, prov))) + .await?; Ok(()) } #[instrument(skip_all)] - fn apply_used( + async fn apply_used( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, namespace: &NamespaceId, usage: &Usage, ) -> Result<(), StoreError> { - let storedactivity = self.activity_by_activity_external_id_and_namespace( - connection, - usage.activity_id.external_id_part(), - namespace, - )?; - - let storedentity = self.entity_by_entity_external_id_and_namespace( - connection, - usage.entity_id.external_id_part(), - namespace, - )?; + let storedactivity = self + .activity_by_activity_external_id_and_namespace( + connection, + usage.activity_id.external_id_part(), + namespace, + ) + .await?; + + let storedentity = self + .entity_by_entity_external_id_and_namespace( + connection, + usage.entity_id.external_id_part(), + namespace, + ) + .await?; use schema::usage::dsl as link; diesel::insert_into(schema::usage::table) @@ -480,30 +526,35 @@ impl Store { &link::entity_id.eq(storedentity.id), )) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } #[instrument(skip_all)] - fn apply_was_informed_by( + async fn apply_was_informed_by( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, namespace: &NamespaceId, activity_id: &ActivityId, informing_activity_id: &ActivityId, ) -> Result<(), StoreError> { - let storedactivity = self.activity_by_activity_external_id_and_namespace( - connection, - activity_id.external_id_part(), - namespace, - )?; - - let storedinformingactivity = self.activity_by_activity_external_id_and_namespace( - connection, - informing_activity_id.external_id_part(), - namespace, - )?; + let storedactivity = self + .activity_by_activity_external_id_and_namespace( + connection, + activity_id.external_id_part(), + namespace, + ) + .await?; + + let storedinformingactivity = self + .activity_by_activity_external_id_and_namespace( + connection, + informing_activity_id.external_id_part(), + namespace, + ) + .await?; use schema::wasinformedby::dsl as link; diesel::insert_into(schema::wasinformedby::table) @@ -512,29 +563,34 @@ impl Store { &link::informing_activity_id.eq(storedinformingactivity.id), )) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } #[instrument(skip(self, connection))] - fn apply_was_associated_with( + async fn apply_was_associated_with( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, namespaceid: &common::prov::NamespaceId, association: &Association, ) -> Result<(), StoreError> { - let storedactivity = self.activity_by_activity_external_id_and_namespace( - connection, - association.activity_id.external_id_part(), - namespaceid, - )?; - - let storedagent = self.agent_by_agent_external_id_and_namespace( - connection, - association.agent_id.external_id_part(), - namespaceid, - )?; + let storedactivity = self + .activity_by_activity_external_id_and_namespace( + connection, + association.activity_id.external_id_part(), + namespaceid, + ) + .await?; + + let storedagent = self + .agent_by_agent_external_id_and_namespace( + connection, + association.agent_id.external_id_part(), + namespaceid, + ) + .await?; use schema::association::dsl as asoc; let no_role = common::prov::Role("".to_string()); @@ -545,29 +601,34 @@ impl Store { &asoc::role.eq(association.role.as_ref().unwrap_or(&no_role)), )) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } #[instrument(skip(self, connection, namespace))] - fn apply_delegation( + async fn apply_delegation( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, namespace: &common::prov::NamespaceId, delegation: &Delegation, ) -> Result<(), StoreError> { - let responsible = self.agent_by_agent_external_id_and_namespace( - connection, - delegation.responsible_id.external_id_part(), - namespace, - )?; - - let delegate = self.agent_by_agent_external_id_and_namespace( - connection, - delegation.delegate_id.external_id_part(), - namespace, - )?; + let responsible = self + .agent_by_agent_external_id_and_namespace( + connection, + delegation.responsible_id.external_id_part(), + namespace, + ) + .await?; + + let delegate = self + .agent_by_agent_external_id_and_namespace( + connection, + delegation.delegate_id.external_id_part(), + namespace, + ) + .await?; let activity = { if let Some(ref activity_id) = delegation.activity_id { @@ -576,7 +637,8 @@ impl Store { connection, activity_id.external_id_part(), namespace, - )? + ) + .await? .id, ) } else { @@ -594,41 +656,46 @@ impl Store { &link::role.eq(delegation.role.as_ref().unwrap_or(&no_role)), )) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } #[instrument(skip(self, connection, namespace))] - fn apply_derivation( + async fn apply_derivation( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, namespace: &common::prov::NamespaceId, derivation: &Derivation, ) -> Result<(), StoreError> { - let stored_generated = self.entity_by_entity_external_id_and_namespace( - connection, - derivation.generated_id.external_id_part(), - namespace, - )?; - - let stored_used = self.entity_by_entity_external_id_and_namespace( - connection, - derivation.used_id.external_id_part(), - namespace, - )?; - - let stored_activity = derivation - .activity_id - .as_ref() - .map(|activity_id| { + let stored_generated = self + .entity_by_entity_external_id_and_namespace( + connection, + derivation.generated_id.external_id_part(), + namespace, + ) + .await?; + + let stored_used = self + .entity_by_entity_external_id_and_namespace( + connection, + derivation.used_id.external_id_part(), + namespace, + ) + .await?; + + let stored_activity = match derivation.activity_id { + Some(ref activity_id) => Some( self.activity_by_activity_external_id_and_namespace( connection, activity_id.external_id_part(), namespace, ) - }) - .transpose()?; + .await?, + ), + None => None, + }; use schema::derivation::dsl as link; diesel::insert_into(schema::derivation::table) @@ -639,29 +706,34 @@ impl Store { &link::activity_id.eq(stored_activity.map_or(-1, |activity| activity.id)), )) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } #[instrument(skip_all)] - fn apply_was_generated_by( + async fn apply_was_generated_by( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, namespace: &common::prov::NamespaceId, generation: &Generation, ) -> Result<(), StoreError> { - let storedactivity = self.activity_by_activity_external_id_and_namespace( - connection, - generation.activity_id.external_id_part(), - namespace, - )?; - - let storedentity = self.entity_by_entity_external_id_and_namespace( - connection, - generation.generated_id.external_id_part(), - namespace, - )?; + let storedactivity = self + .activity_by_activity_external_id_and_namespace( + connection, + generation.activity_id.external_id_part(), + namespace, + ) + .await?; + + let storedentity = self + .entity_by_entity_external_id_and_namespace( + connection, + generation.generated_id.external_id_part(), + namespace, + ) + .await?; use schema::generation::dsl as link; diesel::insert_into(schema::generation::table) @@ -670,29 +742,34 @@ impl Store { &link::generated_entity_id.eq(storedentity.id), )) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } #[instrument(skip(self, connection))] - fn apply_was_attributed_to( + async fn apply_was_attributed_to( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, namespace_id: &common::prov::NamespaceId, attribution: &Attribution, ) -> Result<(), StoreError> { - let stored_entity = self.entity_by_entity_external_id_and_namespace( - connection, - attribution.entity_id.external_id_part(), - namespace_id, - )?; - - let stored_agent = self.agent_by_agent_external_id_and_namespace( - connection, - attribution.agent_id.external_id_part(), - namespace_id, - )?; + let stored_entity = self + .entity_by_entity_external_id_and_namespace( + connection, + attribution.entity_id.external_id_part(), + namespace_id, + ) + .await?; + + let stored_agent = self + .agent_by_agent_external_id_and_namespace( + connection, + attribution.agent_id.external_id_part(), + namespace_id, + ) + .await?; use schema::attribution::dsl as attr; let no_role = common::prov::Role("".to_string()); @@ -703,51 +780,51 @@ impl Store { &attr::role.eq(attribution.role.as_ref().unwrap_or(&no_role)), )) .on_conflict_do_nothing() - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } - pub fn connection( + pub async fn connection( &self, - ) -> Result>, StoreError> { - self.pool.get().map_err(StoreError::DbPool) - } - - #[instrument(skip_all)] - pub fn get_current_agent( - &self, - connection: &mut PgConnection, - ) -> Result { - use schema::agent::dsl; - Ok(schema::agent::table - .filter(dsl::current.ne(0)) - .first::(connection)?) + ) -> Result< + diesel_async::pooled_connection::deadpool::Object, + StoreError, + > { + self.pool.get().await.map_err(StoreError::DbPool) } /// Get the last fully synchronized offset #[instrument(skip_all)] - pub fn get_last_block_id(&self) -> Result, StoreError> { + pub async fn get_last_block_id(&self) -> Result, StoreError> { use schema::ledgersync::dsl; - self.connection()?.build_transaction().run(|connection| { - let block_id_and_tx = schema::ledgersync::table - .order_by(dsl::sync_time) - .select((dsl::bc_offset, dsl::tx_id)) - .first::<(Option, String)>(connection) - .map_err(StoreError::from)?; - - if let Some(block_id) = block_id_and_tx.0 { - Ok(Some(BlockId::try_from(&*block_id)?)) - } else { - Ok(None) - } - }) + self.connection() + .await? + .build_transaction() + .run(|connection| { + Box::pin(async move { + let block_id_and_tx = schema::ledgersync::table + .order_by(dsl::sync_time) + .select((dsl::bc_offset, dsl::tx_id)) + .first::<(Option, String)>(connection) + .await + .map_err(StoreError::from)?; + + if let Some(block_id) = block_id_and_tx.0 { + Ok(Some(BlockId::try_from(&*block_id)?)) + } else { + Ok(None) + } + }) + }) + .await } #[instrument(skip_all)] - pub fn namespace_by_external_id( + pub async fn namespace_by_external_id( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, namespace: &ExternalId, ) -> Result<(NamespaceId, i32), StoreError> { use self::schema::namespace::dsl; @@ -756,6 +833,7 @@ impl Store { .filter(dsl::external_id.eq(namespace)) .select((dsl::id, dsl::external_id, dsl::uuid)) .first::<(i32, String, String)>(connection) + .await .optional()? .ok_or(StoreError::RecordNotFound {})?; @@ -763,23 +841,24 @@ impl Store { } #[instrument(skip_all)] - pub fn new(pool: Pool>) -> Result { + pub fn new(pool: Pool) -> Result { Ok(Store { pool }) } #[instrument(skip_all)] - pub fn populate_prov_model_for_agent( + pub async fn populate_prov_model_for_agent( &self, agent: query::Agent, namespaceid: &NamespaceId, model: &mut ProvModel, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, ) -> Result<(), StoreError> { debug!(?agent, "Map agent to prov"); let attributes = schema::agent_attribute::table .filter(schema::agent_attribute::agent_id.eq(&agent.id)) - .load::(connection)?; + .load::(connection) + .await?; let agentid: AgentId = AgentId::from_external_id(&agent.external_id); model.agents.insert( @@ -816,7 +895,8 @@ impl Store { schema::activity::external_id, schema::delegation::role, )) - .load::<(String, String, String)>(connection)? + .load::<(String, String, String)>(connection) + .await? { model.qualified_delegation( namespaceid, @@ -843,14 +923,13 @@ impl Store { } #[instrument(skip_all)] - pub fn populate_prov_model_for_activity( + pub async fn populate_prov_model_for_activity( &self, activity: query::Activity, namespaceid: &NamespaceId, model: &mut ProvModel, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, ) -> Result<(), StoreError> { - use diesel::prelude::*; use schema::{ activity::dsl as activity_dsl, activity_attribute::dsl as activity_attr_dsl, agent::dsl as agent_dsl, association::dsl as association_dsl, @@ -860,7 +939,8 @@ impl Store { let attributes = activity_attr_dsl::activity_attribute .filter(activity_attr_dsl::activity_id.eq(&activity.id)) - .load::(connection)?; + .load::(connection) + .await?; let id: ActivityId = ActivityId::from_external_id(&activity.external_id); model.activities.insert( @@ -895,13 +975,15 @@ impl Store { .order(generation_dsl::activity_id.asc()) .inner_join(entity_dsl::entity) .select(entity_dsl::external_id) - .load::(connection)?, + .load::(connection) + .await?, usage_dsl::usage .filter(usage_dsl::activity_id.eq(activity.id)) .order(usage_dsl::activity_id.asc()) .inner_join(entity_dsl::entity) .select(entity_dsl::external_id) - .load::(connection)?, + .load::(connection) + .await?, wasinformedby_dsl::wasinformedby .filter(wasinformedby_dsl::activity_id.eq(activity.id)) .inner_join( @@ -909,13 +991,15 @@ impl Store { .on(wasinformedby_dsl::informing_activity_id.eq(activity_dsl::id)), ) .select(activity_dsl::external_id) - .load::(connection)?, + .load::(connection) + .await?, association_dsl::association .filter(association_dsl::activity_id.eq(activity.id)) .order(association_dsl::activity_id.asc()) .inner_join(agent_dsl::agent) .select((agent_dsl::external_id, association_dsl::role)) - .load::<(String, String)>(connection)?, + .load::<(String, String)>(connection) + .await?, ); for generation in generations { @@ -954,12 +1038,12 @@ impl Store { } #[instrument(skip_all)] - pub fn populate_prov_model_for_entity( + pub async fn populate_prov_model_for_entity( &self, entity: query::Entity, namespace_id: &NamespaceId, model: &mut ProvModel, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, ) -> Result<(), StoreError> { let query::Entity { id, namespace_id: _, domaintype, external_id } = entity; @@ -970,7 +1054,8 @@ impl Store { .order(schema::attribution::entity_id.asc()) .inner_join(schema::agent::table) .select((schema::agent::external_id, schema::attribution::role)) - .load::<(String, String)>(connection)? + .load::<(String, String)>(connection) + .await? { model.qualified_attribution( namespace_id, @@ -988,7 +1073,8 @@ impl Store { let attributes = schema::entity_attribute::table .filter(schema::entity_attribute::entity_id.eq(&id)) - .load::(connection)?; + .load::(connection) + .await?; model.entities.insert( (namespace_id.clone(), entity_id.clone()), @@ -1025,7 +1111,8 @@ impl Store { schema::entity::external_id, schema::derivation::typ, )) - .load::<(i32, String, String, i32)>(connection)? + .load::<(i32, String, String, i32)>(connection) + .await? { let typ = DerivationType::try_from(typ) .map_err(|_| StoreError::InvalidDerivationTypeRecord(typ))?; @@ -1048,37 +1135,43 @@ impl Store { } #[instrument(skip_all)] - pub fn load_prov_model_for_namespace( + pub async fn load_prov_model_for_namespace( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, namespace: &NamespaceId, ) -> Result { let mut model = ProvModel::default(); let (namespaceid, nsid) = - self.namespace_by_external_id(connection, namespace.external_id_part())?; + self.namespace_by_external_id(connection, namespace.external_id_part()).await?; let agents = schema::agent::table .filter(schema::agent::namespace_id.eq(&nsid)) - .load::(connection)?; + .load::(connection) + .await?; for agent in agents { - self.populate_prov_model_for_agent(agent, &namespaceid, &mut model, connection)?; + self.populate_prov_model_for_agent(agent, &namespaceid, &mut model, connection) + .await?; } let activities = schema::activity::table .filter(schema::activity::namespace_id.eq(nsid)) - .load::(connection)?; + .load::(connection) + .await?; for activity in activities { - self.populate_prov_model_for_activity(activity, &namespaceid, &mut model, connection)?; + self.populate_prov_model_for_activity(activity, &namespaceid, &mut model, connection) + .await?; } let entities = schema::entity::table .filter(schema::entity::namespace_id.eq(nsid)) - .load::(connection)?; + .load::(connection) + .await?; for entity in entities { - self.populate_prov_model_for_entity(entity, &namespaceid, &mut model, connection)?; + self.populate_prov_model_for_entity(entity, &namespaceid, &mut model, connection) + .await?; } Ok(model) @@ -1086,56 +1179,67 @@ impl Store { /// Set the last fully synchronized offset #[instrument(skip(self), level = "info")] - pub fn set_last_block_id( + pub async fn set_last_block_id( &self, block_id: &BlockId, tx_id: ChronicleTransactionId, ) -> Result<(), StoreError> { use schema::ledgersync as dsl; - Ok(self.connection()?.build_transaction().run(|connection| { - diesel::insert_into(dsl::table) - .values(( - dsl::bc_offset.eq(block_id.to_string()), - dsl::tx_id.eq(&*tx_id.to_string()), - (dsl::sync_time.eq(Utc::now().naive_utc())), - )) - .on_conflict(dsl::tx_id) - .do_update() - .set(dsl::sync_time.eq(Utc::now().naive_utc())) - .execute(connection) - .map(|_| ()) - })?) + self.connection() + .await? + .build_transaction() + .run(|connection| { + Box::pin(async move { + diesel::insert_into(dsl::table) + .values(( + dsl::bc_offset.eq(block_id.to_string()), + dsl::tx_id.eq(&*tx_id.to_string()), + (dsl::sync_time.eq(Utc::now().naive_utc())), + )) + .on_conflict(dsl::tx_id) + .do_update() + .set(dsl::sync_time.eq(Utc::now().naive_utc())) + .execute(connection) + .await + .map(|_| ()) + }) + }) + .await?; + + Ok(()) } #[instrument(skip_all)] - pub fn apply_use_agent( + pub async fn apply_use_agent( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, external_id: &ExternalId, namespace: &ExternalId, ) -> Result<(), StoreError> { - let (_, nsid) = self.namespace_by_external_id(connection, namespace)?; + let (_, nsid) = self.namespace_by_external_id(connection, namespace).await?; use schema::agent::dsl; diesel::update(schema::agent::table.filter(dsl::current.ne(0))) .set(dsl::current.eq(0)) - .execute(connection)?; + .execute(connection) + .await?; diesel::update( schema::agent::table .filter(dsl::external_id.eq(external_id).and(dsl::namespace_id.eq(nsid))), ) .set(dsl::current.eq(1)) - .execute(connection)?; + .execute(connection) + .await?; Ok(()) } #[instrument(skip_all)] - pub fn prov_model_for_agent_id( + pub async fn prov_model_for_agent_id( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, id: &AgentId, ns: &ExternalId, ) -> Result { @@ -1144,19 +1248,21 @@ impl Store { .filter(schema::agent::external_id.eq(id.external_id_part())) .filter(schema::namespace::external_id.eq(ns)) .select(query::Agent::as_select()) - .first(connection)?; + .first(connection) + .await?; - let namespace = self.namespace_by_external_id(connection, ns)?.0; + let namespace = self.namespace_by_external_id(connection, ns).await?.0; let mut model = ProvModel::default(); - self.populate_prov_model_for_agent(agent, &namespace, &mut model, connection)?; + self.populate_prov_model_for_agent(agent, &namespace, &mut model, connection) + .await?; Ok(model) } #[instrument(skip_all)] - pub fn apply_prov_model_for_agent_id( + pub async fn apply_prov_model_for_agent_id( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, mut model: ProvModel, id: &AgentId, ns: &ExternalId, @@ -1167,18 +1273,20 @@ impl Store { .filter(schema::namespace::external_id.eq(ns)) .select(query::Agent::as_select()) .first(connection) + .await .optional()? { - let namespace = self.namespace_by_external_id(connection, ns)?.0; - self.populate_prov_model_for_agent(agent, &namespace, &mut model, connection)?; + let namespace = self.namespace_by_external_id(connection, ns).await?.0; + self.populate_prov_model_for_agent(agent, &namespace, &mut model, connection) + .await?; } Ok(model) } #[instrument(skip_all)] - pub fn prov_model_for_activity_id( + pub async fn prov_model_for_activity_id( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, id: &ActivityId, ns: &ExternalId, ) -> Result { @@ -1187,19 +1295,21 @@ impl Store { .filter(schema::activity::external_id.eq(id.external_id_part())) .filter(schema::namespace::external_id.eq(ns)) .select(query::Activity::as_select()) - .first(connection)?; + .first(connection) + .await?; - let namespace = self.namespace_by_external_id(connection, ns)?.0; + let namespace = self.namespace_by_external_id(connection, ns).await?.0; let mut model = ProvModel::default(); - self.populate_prov_model_for_activity(activity, &namespace, &mut model, connection)?; + self.populate_prov_model_for_activity(activity, &namespace, &mut model, connection) + .await?; Ok(model) } #[instrument(skip_all)] - pub fn apply_prov_model_for_activity_id( + pub async fn apply_prov_model_for_activity_id( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, mut model: ProvModel, id: &ActivityId, ns: &ExternalId, @@ -1210,18 +1320,20 @@ impl Store { .filter(schema::namespace::external_id.eq(ns)) .select(query::Activity::as_select()) .first(connection) + .await .optional()? { - let namespace = self.namespace_by_external_id(connection, ns)?.0; - self.populate_prov_model_for_activity(activity, &namespace, &mut model, connection)?; + let namespace = self.namespace_by_external_id(connection, ns).await?.0; + self.populate_prov_model_for_activity(activity, &namespace, &mut model, connection) + .await?; } Ok(model) } #[instrument(skip_all)] - pub fn prov_model_for_entity_id( + pub async fn prov_model_for_entity_id( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, id: &EntityId, ns: &ExternalId, ) -> Result { @@ -1230,19 +1342,21 @@ impl Store { .filter(schema::entity::external_id.eq(id.external_id_part())) .filter(schema::namespace::external_id.eq(ns)) .select(query::Entity::as_select()) - .first(connection)?; + .first(connection) + .await?; - let namespace = self.namespace_by_external_id(connection, ns)?.0; + let namespace = self.namespace_by_external_id(connection, ns).await?.0; let mut model = ProvModel::default(); - self.populate_prov_model_for_entity(entity, &namespace, &mut model, connection)?; + self.populate_prov_model_for_entity(entity, &namespace, &mut model, connection) + .await?; Ok(model) } #[instrument(skip_all)] - pub fn apply_prov_model_for_entity_id( + pub async fn apply_prov_model_for_entity_id( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, mut model: ProvModel, id: &EntityId, ns: &ExternalId, @@ -1253,18 +1367,20 @@ impl Store { .filter(schema::namespace::external_id.eq(ns)) .select(query::Entity::as_select()) .first(connection) + .await .optional()? { - let namespace = self.namespace_by_external_id(connection, ns)?.0; - self.populate_prov_model_for_entity(entity, &namespace, &mut model, connection)?; + let namespace = self.namespace_by_external_id(connection, ns).await?.0; + self.populate_prov_model_for_entity(entity, &namespace, &mut model, connection) + .await?; } Ok(model) } #[instrument(skip_all)] - pub fn prov_model_for_usage( + pub async fn prov_model_for_usage( &self, - connection: &mut PgConnection, + connection: &mut AsyncPgConnection, mut model: ProvModel, id: &EntityId, activity_id: &ActivityId, @@ -1276,6 +1392,7 @@ impl Store { .filter(schema::namespace::external_id.eq(ns)) .select(query::Entity::as_select()) .first(connection) + .await .optional()? { if let Some(activity) = schema::activity::table @@ -1284,22 +1401,24 @@ impl Store { .filter(schema::namespace::external_id.eq(ns)) .select(query::Activity::as_select()) .first(connection) + .await .optional()? { - let namespace = self.namespace_by_external_id(connection, ns)?.0; + let namespace = self.namespace_by_external_id(connection, ns).await?.0; for used in schema::usage::table .filter(schema::usage::activity_id.eq(activity.id)) .order(schema::usage::activity_id.asc()) .inner_join(schema::entity::table) .select(schema::entity::external_id) - .load::(connection)? + .load::(connection) + .await? { model.used(namespace.clone(), activity_id, &EntityId::from_external_id(used)); } - self.populate_prov_model_for_entity(entity, &namespace, &mut model, connection)?; - self.populate_prov_model_for_activity( - activity, &namespace, &mut model, connection, - )?; + self.populate_prov_model_for_entity(entity, &namespace, &mut model, connection) + .await?; + self.populate_prov_model_for_activity(activity, &namespace, &mut model, connection) + .await?; } } Ok(model) diff --git a/crates/chronicle-persistence/src/queryable.rs b/crates/chronicle-persistence/src/queryable.rs index 9dfc9780..a86fdc87 100644 --- a/crates/chronicle-persistence/src/queryable.rs +++ b/crates/chronicle-persistence/src/queryable.rs @@ -2,7 +2,7 @@ use async_graphql::{Object, SimpleObject}; use chrono::NaiveDateTime; use diesel::{Queryable, Selectable}; -#[derive(Default, Queryable, Selectable, SimpleObject)] +#[derive(Default, Queryable, Selectable, SimpleObject, Clone)] #[diesel(table_name = crate::schema::agent)] pub struct Agent { pub id: i32, diff --git a/crates/chronicle-telemetry/src/telemetry.rs b/crates/chronicle-telemetry/src/telemetry.rs index 96a9a48b..8d1f098f 100644 --- a/crates/chronicle-telemetry/src/telemetry.rs +++ b/crates/chronicle-telemetry/src/telemetry.rs @@ -3,6 +3,10 @@ use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; use opentelemetry_sdk::{runtime, trace as sdktrace}; use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer}; use tracing_subscriber::{fmt::format::FmtSpan, prelude::*, EnvFilter}; +use std::sync::atomic::{AtomicBool, Ordering}; + +// Global state to ensure telemetry is only initialized once +static TELEMETRY_INITIALIZED: AtomicBool = AtomicBool::new(false); fn init_tracer_provider() -> Result { opentelemetry_otlp::new_pipeline() @@ -33,6 +37,11 @@ pub enum ConsoleLogging { } pub fn telemetry(console_logging: ConsoleLogging) { + // Check if telemetry has already been initialized + if TELEMETRY_INITIALIZED.swap(true, Ordering::SeqCst) { + return; // Telemetry has already been initialized, so return early + } + let result = init_tracer_provider(); assert!(result.is_ok(), "Init tracer failed with error: {:?}", result.err()); let tracer = result.unwrap(); diff --git a/crates/chronicle-test-infrastructure/Cargo.toml b/crates/chronicle-test-infrastructure/Cargo.toml index e05ad809..f5e6fb3e 100644 --- a/crates/chronicle-test-infrastructure/Cargo.toml +++ b/crates/chronicle-test-infrastructure/Cargo.toml @@ -18,6 +18,8 @@ common = { path = "../common", features = [ "std", ] } diesel = { workspace = true } +diesel-async = { workspace = true } +diesel_migrations = { workspace = true } frame-support = { git = 'https://github.com/paritytech/polkadot-sdk.git', tag = 'polkadot-v1.9.0', features = ["std"] } frame-system = { git = 'https://github.com/paritytech/polkadot-sdk.git', tag = 'polkadot-v1.9.0', features = ["std"] } sp-core = { git = 'https://github.com/paritytech/polkadot-sdk.git', tag = 'polkadot-v1.9.0' } diff --git a/crates/chronicle-test-infrastructure/src/api_test.rs b/crates/chronicle-test-infrastructure/src/api_test.rs index ecd0a14f..79ee7926 100644 --- a/crates/chronicle-test-infrastructure/src/api_test.rs +++ b/crates/chronicle-test-infrastructure/src/api_test.rs @@ -374,61 +374,6 @@ async fn start_activity() { "test": "test" } } - "###); - - api.dispatch( - ApiCommand::Agent(AgentCommand::UseInContext { - id: AgentId::from_external_id("testagent"), - namespace: "testns".into(), - }), - identity.clone(), - ) - .await - .unwrap(); - - insta::assert_json_snapshot!( - api.dispatch(ApiCommand::Activity(ActivityCommand::Start { - id: ActivityId::from_external_id("testactivity"), - namespace: "testns".into(), - time: Some(Utc.with_ymd_and_hms(2014, 7, 8, 9, 10, 11).unwrap()), - agent: None, - }), identity) - .await - .unwrap() - .unwrap() - .0 - .to_json() - .compact_stable_order() - .await - .unwrap(), @r###" - { - "@context": "http://chronicle.works/chr/1.0/c.jsonld", - "@graph": [ - { - "@id": "chronicle:activity:testactivity", - "@type": "prov:Activity", - "externalId": "testactivity", - "namespace": "chronicle:ns:testns:11111111-1111-1111-1111-111111111111", - "prov:qualifiedAssociation": { - "@id": "chronicle:association:testagent:testactivity:role=" - }, - "startTime": "2014-07-08T09:10:11+00:00", - "value": {}, - "wasAssociatedWith": [ - "chronicle:agent:testagent" - ] - }, - { - "@id": "chronicle:association:testagent:testactivity:role=", - "@type": "prov:Association", - "agent": "chronicle:agent:testagent", - "namespace": "chronicle:ns:testns:11111111-1111-1111-1111-111111111111", - "prov:hadActivity": { - "@id": "chronicle:activity:testactivity" - } - } - ] - } "###); } @@ -544,22 +489,12 @@ async fn contradict_start_time() { } "###); - api.dispatch( - ApiCommand::Agent(AgentCommand::UseInContext { - id: AgentId::from_external_id("testagent"), - namespace: "testns".into(), - }), - identity.clone(), - ) - .await - .unwrap(); - insta::assert_json_snapshot!( api.dispatch(ApiCommand::Activity(ActivityCommand::Start { id: ActivityId::from_external_id("testactivity"), namespace: "testns".into(), time: Some(Utc.with_ymd_and_hms(2014, 7, 8, 9, 10, 11).unwrap()), - agent: None, + agent: Some(AgentId::from_external_id("testagent")), }), identity.clone()) .await .unwrap() @@ -660,22 +595,12 @@ async fn contradict_end_time() { } "###); - api.dispatch( - ApiCommand::Agent(AgentCommand::UseInContext { - id: AgentId::from_external_id("testagent"), - namespace: "testns".into(), - }), - identity.clone(), - ) - .await - .unwrap(); - insta::assert_json_snapshot!( api.dispatch(ApiCommand::Activity(ActivityCommand::End { id: ActivityId::from_external_id("testactivity"), namespace: "testns".into(), time: Some(Utc.with_ymd_and_hms(2018, 7, 8, 9, 10, 11).unwrap()), - agent: None, + agent: Some(AgentId::from_external_id("testagent")), }), identity.clone()) .await .unwrap() @@ -776,22 +701,12 @@ async fn end_activity() { } "###); - api.dispatch( - ApiCommand::Agent(AgentCommand::UseInContext { - id: AgentId::from_external_id("testagent"), - namespace: "testns".into(), - }), - identity.clone(), - ) - .await - .unwrap(); - insta::assert_json_snapshot!( api.dispatch(ApiCommand::Activity(ActivityCommand::Start { id: ActivityId::from_external_id("testactivity"), namespace: "testns".into(), time: Some(Utc.with_ymd_and_hms(2014, 7, 8, 9, 10, 11).unwrap()), - agent: None, + agent: Some(AgentId::from_external_id("testagent")), }), identity.clone()) .await .unwrap() @@ -924,16 +839,6 @@ async fn activity_use() { } "###); - api.dispatch( - ApiCommand::Agent(AgentCommand::UseInContext { - id: AgentId::from_external_id("testagent"), - namespace: "testns".into(), - }), - identity.clone(), - ) - .await - .unwrap(); - insta::assert_json_snapshot!( api.dispatch(ApiCommand::Activity(ActivityCommand::Create { id: "testactivity".into(), diff --git a/crates/chronicle-test-infrastructure/src/substitutes/mod.rs b/crates/chronicle-test-infrastructure/src/substitutes/mod.rs index d77442d9..83d183f1 100644 --- a/crates/chronicle-test-infrastructure/src/substitutes/mod.rs +++ b/crates/chronicle-test-infrastructure/src/substitutes/mod.rs @@ -1,10 +1,12 @@ mod mockchain; mod stubstrate; +use std::path::Path; + use crate::substitutes::stubstrate::Stubstrate; use api::{ commands::{ApiCommand, ApiResponse}, - ApiError, + ApiError, StoreError, }; use common::{ identity::AuthId, @@ -19,9 +21,9 @@ use chronicle_signing::{ CHRONICLE_NAMESPACE, }; -use diesel::{ - r2d2::{ConnectionManager, Pool}, - Connection, PgConnection, +use diesel_async::{ + pooled_connection::{deadpool::Pool, AsyncDieselConnectionManager}, + AsyncPgConnection, }; use testcontainers::{images::postgres::Postgres, Container}; @@ -34,7 +36,7 @@ lazy_static! { } pub struct TemporaryDatabase<'a> { - db_uris: Vec, + db_uri: String, container: Container<'a, Postgres>, } @@ -46,25 +48,51 @@ impl<'a> Drop for TemporaryDatabase<'a> { } impl<'a> TemporaryDatabase<'a> { - pub fn connection_pool(&self) -> Result>, r2d2::Error> { - let db_uri = self - .db_uris - .iter() - .find(|db_uri| PgConnection::establish(db_uri).is_ok()) - .expect("cannot establish connection"); - Pool::builder().build(ConnectionManager::::new(db_uri)) + pub fn connection_pool(&self) -> Result, StoreError> { + use diesel::{pg::PgConnection, Connection}; + use diesel_migrations::MigrationHarness; + + // Create a sync connection to run migrations + let mut conn = PgConnection::establish(&self.db_uri).map_err(StoreError::DbConnection)?; + + // Run embedded migrations + conn.run_pending_migrations(chronicle::persistence::MIGRATIONS) + .map_err(StoreError::DbMigration)?; + + let config = + AsyncDieselConnectionManager::::new(&self.db_uri); + let pool = Pool::builder(config).max_size(30).build()?; + Ok(pool) } } +pub fn is_running_in_container() -> bool { + Path::new("/.dockerenv").exists() +} + impl<'a> Default for TemporaryDatabase<'a> { fn default() -> Self { let container = CLIENT.run(Postgres::default()); - const PORT: u16 = 5432; + let port = if is_running_in_container() { + 5432 + } else { + + container.get_host_port_ipv4(5432) + }; + + let ip = if is_running_in_container() { + container.get_bridge_ip_address().to_string() + + } else { + "127.0.0.1".to_owned() + }; + Self { - db_uris: vec![ - format!("postgresql://postgres@127.0.0.1:{}/", container.get_host_port_ipv4(PORT)), - format!("postgresql://postgres@{}:{}/", container.get_bridge_ip_address(), PORT), - ], + db_uri: format!( + "postgresql://postgres@{}:{}/", + ip, + port + ), container, } } @@ -150,10 +178,9 @@ pub async fn test_api<'a>() -> TestDispatch<'a> { let liveness_check_interval = None; - let dispatch = Api::new( + let dispatch = Api::create_dispatch( pool, embed_substrate.clone(), - SameUuid, secrets, vec![NamespaceId::from_external_id( "testns", diff --git a/crates/chronicle/Cargo.toml b/crates/chronicle/Cargo.toml index a8e412d0..d13f0441 100644 --- a/crates/chronicle/Cargo.toml +++ b/crates/chronicle/Cargo.toml @@ -19,6 +19,8 @@ clap_complete = { workspace = true } colored_json = { workspace = true } const_format = { workspace = true } diesel = { workspace = true } +diesel-async = { workspace = true } +diesel_migrations = { workspace = true } dotenvy = { workspace = true } futures = { workspace = true } genco = { workspace = true } diff --git a/crates/chronicle/src/bootstrap/cli.rs b/crates/chronicle/src/bootstrap/cli.rs index 9a2c21e6..f0c09511 100644 --- a/crates/chronicle/src/bootstrap/cli.rs +++ b/crates/chronicle/src/bootstrap/cli.rs @@ -399,24 +399,7 @@ impl SubCommand for AgentCliModel { define = define.arg(attr.as_arg()); } - cmd.subcommand(define).subcommand( - Command::new("use") - .about("Make the specified agent the context for activities and entities") - .arg( - Arg::new("id") - .help("A valid chronicle agent IRI") - .required(true) - .takes_value(true), - ) - .arg( - Arg::new("namespace") - .short('n') - .long("namespace") - .default_value("default") - .required(false) - .takes_value(true), - ), - ) + define } fn matches(&self, matches: &ArgMatches) -> Result, CliError> { @@ -428,13 +411,6 @@ impl SubCommand for AgentCliModel { }))); } - if let Some(matches) = matches.subcommand_matches("use") { - return Ok(Some(ApiCommand::Agent(AgentCommand::UseInContext { - id: id_from(matches, "id")?, - namespace: namespace_from(matches)?, - }))); - }; - Ok(None) } } @@ -950,6 +926,13 @@ impl SubCommand for CliModel { .env("ARROW_LISTEN_SOCKET") .help("The arrow flight address"), ) + .arg( + Arg::new("liveness-probe") + .long("liveness-probe") + .takes_value(true) + .default_missing_value("30s") + .help("The frequency of liveness probe") + ) .arg( Arg::new("interface") .long("interface") @@ -964,7 +947,7 @@ impl SubCommand for CliModel { .alias("open") .required(false) .takes_value(false) - .help("Deprecated option (after v0.6.0) to make available the GraphQL Playground"), + .help("Make the GraphQL Playground available at /"), ).arg( Arg::new("require-auth") .long("require-auth") diff --git a/crates/chronicle/src/bootstrap/mod.rs b/crates/chronicle/src/bootstrap/mod.rs index 9ff6115b..62ed13d6 100644 --- a/crates/chronicle/src/bootstrap/mod.rs +++ b/crates/chronicle/src/bootstrap/mod.rs @@ -8,9 +8,10 @@ use std::{ use async_graphql::ObjectType; use clap::{ArgMatches, Command}; use clap_complete::{generate, Generator, Shell}; -use diesel::{ - r2d2::{ConnectionManager, Pool}, - PgConnection, + +use diesel_async::{ + pooled_connection::{deadpool::Pool, AsyncDieselConnectionManager}, + AsyncPgConnection, }; use futures::{Future, FutureExt, StreamExt}; use tracing::{debug, error, info, instrument, warn}; @@ -25,7 +26,10 @@ use api::{ commands::ApiResponse, Api, ApiDispatch, ApiError, StoreError, UuidGen, }; -use chronicle_persistence::database::{get_connection_with_retry, DatabaseConnector}; +use chronicle_persistence::{ + database::{get_connection_with_retry, DatabaseConnector}, + MIGRATIONS, +}; use chronicle_signing::{ chronicle_secret_names, ChronicleSecretsOptions, ChronicleSigning, BATCHER_NAMESPACE, CHRONICLE_NAMESPACE, @@ -55,12 +59,12 @@ pub mod opa; #[cfg(not(feature = "devmode"))] fn validator_address(options: &ArgMatches) -> Result { - Ok(options + options .value_of("validator") .map(str::to_string) .ok_or(CliError::MissingArgument { arg: "validator".to_owned() }) .and_then(|s| Url::parse(&s).map_err(CliError::from)) - .map_err(CliError::from)?) + .map_err(CliError::from) } #[allow(dead_code)] @@ -91,7 +95,7 @@ struct UniqueUuid; impl UuidGen for UniqueUuid {} -type ConnectionPool = Pool>; +type ConnectionPool = Pool; struct RemoteDatabaseConnector { db_uri: String, @@ -99,10 +103,20 @@ struct RemoteDatabaseConnector { #[async_trait::async_trait] impl DatabaseConnector<(), StoreError> for RemoteDatabaseConnector { - async fn try_connect(&self) -> Result<((), Pool>), StoreError> { - use diesel::Connection; - PgConnection::establish(&self.db_uri)?; - Ok(((), Pool::builder().build(ConnectionManager::::new(&self.db_uri))?)) + async fn try_connect(&self) -> Result<((), ConnectionPool), StoreError> { + use diesel::{pg::PgConnection, Connection}; + use diesel_migrations::MigrationHarness; + + // Create a sync connection to run migrations + let mut conn = PgConnection::establish(&self.db_uri).map_err(StoreError::DbConnection)?; + + // Run embedded migrations + conn.run_pending_migrations(MIGRATIONS).map_err(StoreError::DbMigration)?; + + let config = + AsyncDieselConnectionManager::::new(&self.db_uri); + let pool = Pool::builder(config).max_size(30).build()?; + Ok(((), pool)) } fn should_retry(&self, error: &StoreError) -> bool { @@ -184,7 +198,7 @@ where #[allow(dead_code)] fn namespace_bindings(options: &ArgMatches) -> Vec { options - .get_many::("namespace-binding") + .get_many::("namespace-binding") .map(|values| { values .map(|value| { @@ -269,10 +283,9 @@ pub async fn api( ) -> Result { let ledger = ledger(options).await?; - Ok(Api::new( + Ok(Api::create_dispatch( pool.clone(), ledger, - UniqueUuid, chronicle_signing(options).await?, namespace_bindings(options), policy_address, diff --git a/crates/common/src/ledger.rs b/crates/common/src/ledger.rs index 2544d7b4..ec70cd05 100644 --- a/crates/common/src/ledger.rs +++ b/crates/common/src/ledger.rs @@ -1,5 +1,4 @@ use tracing::instrument; -use uuid::Uuid; use crate::{ identity::SignedIdentity, @@ -22,11 +21,10 @@ use parity_scale_codec::{ #[cfg(not(feature = "std"))] use scale_info::prelude::*; - #[cfg(feature = "std")] use std::{ boxed::Box, collections::btree_map::Entry, collections::BTreeMap, collections::BTreeSet, - sync::Arc + sync::Arc, }; #[derive(Debug, Clone)] @@ -101,14 +99,13 @@ pub struct OperationSubmission { } impl OperationSubmission { - pub fn correlation_id(&self) -> &[u8; 32] { &self.correlation_id } #[cfg(feature = "std")] pub fn logical_hash(operations: &[ChronicleOperation]) -> [u8; 32] { - use k256::sha2::{Sha256, Digest}; + use k256::sha2::{Digest, Sha256}; use tracing::debug; let mut hasher = Sha256::new(); @@ -143,7 +140,6 @@ impl OperationSubmission { } } - pub type SubmitResult = Result; #[derive(Debug, Clone)] diff --git a/crates/common/src/prov/operations.rs b/crates/common/src/prov/operations.rs index b7b9e15c..7215939d 100644 --- a/crates/common/src/prov/operations.rs +++ b/crates/common/src/prov/operations.rs @@ -617,10 +617,7 @@ pub enum ChronicleOperation { WasInformedBy(WasInformedBy), } - - impl ChronicleOperation { - pub fn create_namespace(id: NamespaceId) -> Self { ChronicleOperation::CreateNamespace(CreateNamespace::new(id)) } diff --git a/crates/opactl/src/test/stubstrate.rs b/crates/opactl/src/test/stubstrate.rs index 3a32325a..76510bf0 100644 --- a/crates/opactl/src/test/stubstrate.rs +++ b/crates/opactl/src/test/stubstrate.rs @@ -73,13 +73,18 @@ impl LedgerReader for Stubstrate { ) -> Result>, Self::Error> { tracing::debug!("Starting state updates stream from block {:?}", from_block); let rx = self.tx.subscribe(); - let stream = tokio_stream::wrappers::BroadcastStream::new(rx) - .map(|event| { - let event = event.unwrap(); - let correlation_id = event.correlation_id().into(); - (event, correlation_id, BlockId::Unknown, Position::from(0), Span::NotTraced) - }) - .boxed(); + let stream = tokio_stream::wrappers::BroadcastStream::new(rx).map(|event| { + let event = event.unwrap(); + let correlation_id = event.correlation_id().into(); + (event, correlation_id, BlockId::Unknown, Position::from(0), Span::NotTraced) + }); + + let stream = if let Some(number_of_blocks) = number_of_blocks { + stream.take(number_of_blocks as usize).boxed() + } else { + stream.boxed() + }; + Ok(stream) } } diff --git a/crates/pallet-chronicle/src/tests.rs b/crates/pallet-chronicle/src/tests.rs index dfb922c0..9d688dd9 100644 --- a/crates/pallet-chronicle/src/tests.rs +++ b/crates/pallet-chronicle/src/tests.rs @@ -40,7 +40,7 @@ fn single_operation() { id: NamespaceId::from_external_id("test", uuid), }); - let sub = OperationSubmission::new_anonymous( vec![op.clone()]); + let sub = OperationSubmission::new_anonymous(vec![op.clone()]); // Dispatch our operation assert_ok!(ChronicleModule::apply(RuntimeOrigin::signed(1), sub.clone(),)); diff --git a/crates/protocol-abstract/src/abstract_ledger.rs b/crates/protocol-abstract/src/abstract_ledger.rs index 46beff8f..1b49b917 100644 --- a/crates/protocol-abstract/src/abstract_ledger.rs +++ b/crates/protocol-abstract/src/abstract_ledger.rs @@ -128,10 +128,11 @@ pub trait MessageBuilder {} pub trait LedgerTransaction { type Error: std::error::Error + Send + Sync + 'static; type Payload: Sized + Send + Sync; + const SIGNED: bool = false; + async fn as_payload(&self) -> Result; //A logical hash of the transaction fn correlation_id(&self) -> [u8; 32]; - } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/protocol-substrate-chronicle/src/subxt_client.rs b/crates/protocol-substrate-chronicle/src/subxt_client.rs index c517ea2d..3c79ac1d 100644 --- a/crates/protocol-substrate-chronicle/src/subxt_client.rs +++ b/crates/protocol-substrate-chronicle/src/subxt_client.rs @@ -1,4 +1,4 @@ -use std::{convert::Infallible, hash::Hash, sync::Arc}; +use std::{convert::Infallible, sync::Arc}; use chronicle_signing::{ ChronicleSigning, OwnedSecret, SecretError, BATCHER_NAMESPACE, BATCHER_PK, diff --git a/crates/protocol-substrate-opa/src/transaction.rs b/crates/protocol-substrate-opa/src/transaction.rs index 0a2185a1..1a9fa6e2 100644 --- a/crates/protocol-substrate-opa/src/transaction.rs +++ b/crates/protocol-substrate-opa/src/transaction.rs @@ -1,5 +1,3 @@ -use std::hash::Hash; - use chronicle_signing::{ ChronicleSigning, OwnedSecret, SecretError, BATCHER_NAMESPACE, BATCHER_PK, }; diff --git a/crates/protocol-substrate/Cargo.toml b/crates/protocol-substrate/Cargo.toml index 77ff8418..783e0cdd 100644 --- a/crates/protocol-substrate/Cargo.toml +++ b/crates/protocol-substrate/Cargo.toml @@ -18,6 +18,7 @@ thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } +jsonrpsee-core = { version = "0.23.2" } scale-decode = { version="0.13.1", features = ["derive"] } scale-encode = { version="0.7.1", features = ["derive"] } # Local dependencies diff --git a/crates/protocol-substrate/src/lib.rs b/crates/protocol-substrate/src/lib.rs index 01640e39..ebedba28 100644 --- a/crates/protocol-substrate/src/lib.rs +++ b/crates/protocol-substrate/src/lib.rs @@ -3,7 +3,7 @@ mod subxt_client; pub use subxt::PolkadotConfig; pub use subxt_client::*; -use subxt::{config::signed_extensions, utils::MultiAddress, SubstrateConfig}; +use subxt::{utils::MultiAddress, SubstrateConfig}; /// Default set of commonly used types by Polkadot nodes. pub struct ChronicleConfig; diff --git a/crates/protocol-substrate/src/subxt_client.rs b/crates/protocol-substrate/src/subxt_client.rs index aeb24eda..3e863375 100644 --- a/crates/protocol-substrate/src/subxt_client.rs +++ b/crates/protocol-substrate/src/subxt_client.rs @@ -8,16 +8,9 @@ use futures::{ use pallet_chronicle::ChronicleTransactionId; use subxt::{ backend::BackendExt, - config::{ - signed_extensions::CheckNonceParams, DefaultExtrinsicParamsBuilder, ExtrinsicParamsEncoder, - RefineParamsData, - }, dynamic::Value, error::MetadataError, - ext::{ - codec::{Compact, Encode}, - sp_core::twox_128, - }, + ext::{codec::Encode, sp_core::twox_128}, metadata::{ types::{PalletMetadata, StorageEntryMetadata, StorageEntryType}, DecodeWithMetadata, @@ -80,12 +73,12 @@ where // TODO: bring the pallet / call name in from trait - #[tracing::instrument(level = "debug", skip(self, signer, correlation_id, operations), fields( + #[tracing::instrument(level = "debug", skip(self, _signer, correlation_id, operations), fields( correlation_id = % hex::encode(correlation_id), ret ))] pub async fn create_extrinsic + Send>( &self, - signer: &S, + _signer: &S, correlation_id: [u8; 32], operations: &T, ) -> ExtrinsicResult { @@ -102,12 +95,9 @@ where pub async fn send_extrinsic( &self, consistency: WriteConsistency, - extrinsic: ( - SubmittableExtrinsic>, - [u8; 32], - ), + extrinsic: (SubmittableExtrinsic>, [u8; 32]), ) -> Result { - extrinsic + let tx = extrinsic .0 .submit_and_watch() .and_then(|progress| match consistency { @@ -118,8 +108,29 @@ where .boxed(), }) .await - .map(|_| extrinsic.1.into()) - .map_err(|e| (e, ChronicleTransactionId::from(extrinsic.1))) + .map(|_| extrinsic.1.into()); + + match tx { + Ok(tx) => Ok(tx), + Err(e) => { + if let subxt::Error::Rpc(subxt::error::RpcError::ClientError(err)) = &e { + if let Some(jsonrpsee_core::client::Error::Call(error_object)) = + err.downcast_ref::() + { + let error_code = error_object.code(); + // Swallow 1002 && 1013, logging it as debug + if error_code == 1013 || error_code == 1012 { + tracing::debug!( + "Submission has been successful, ignoring error: {:?}", + err + ); + return Ok(extrinsic.1.into()); + } + } + } + Err((e, ChronicleTransactionId::from(extrinsic.1))) + }, + } } } @@ -211,7 +222,7 @@ where subxt::utils::H256::zero(), ) .await - .map_err(|e| SubxtClientError::from(e)); + .map_err(SubxtClientError::from); match block_hash { Ok(hash) => Some((hash, (client, block_num + 1))),