From c6c4d993f46885a7c98ccd97c616ca63ecb988e1 Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 14 Mar 2024 16:29:29 +0800 Subject: [PATCH 01/20] feat: wip event indexing + snapshot data --- .github/workflows/release-program.yaml | 6 +- Cargo.lock | 1008 +-- Cargo.toml | 2 +- observability/Cargo.lock | 6113 +++++++++++++++++ observability/Cargo.toml | 32 + observability/crates/event_indexer/Cargo.toml | 42 + .../event_indexer/bin/backfill_events.rs | 182 + .../crates/event_indexer/bin/index_events.rs | 74 + .../crates/event_indexer/src/backfiller.rs | 279 + .../crates/event_indexer/src/error.rs | 20 + .../crates/event_indexer/src/indexer.rs | 269 + observability/crates/event_indexer/src/lib.rs | 5 + .../crates/event_indexer/src/parser.rs | 561 ++ .../crates/event_indexer/src/snapshot.rs | 193 + observability/crates/rpc_utils/Cargo.toml | 10 + observability/crates/rpc_utils/src/lib.rs | 58 + observability/etl/dataflow-etls/.dockerignore | 6 - observability/etl/dataflow-etls/.gcloudignore | 7 - observability/etl/dataflow-etls/.gitignore | 7 - observability/etl/dataflow-etls/.mypy.ini | 32 - observability/etl/dataflow-etls/Dockerfile | 24 - observability/etl/dataflow-etls/MANIFEST.in | 1 - observability/etl/dataflow-etls/README.md | 26 - .../dataflow-etls/dataflow_etls/__init__.py | 0 .../dataflow_etls/account_parsing.py | 76 - .../dataflow_etls/idl_versions.py | 94 - .../marginfi-v0.json | 2309 ------- .../marginfi-v0.json | 1904 ----- .../marginfi-v1.json | 1907 ----- .../marginfi-v2.json | 1899 ----- .../marginfi-v3.json | 2032 ------ .../marginfi-v4.json | 2216 ------ .../marginfi-v0.json | 2478 ------- .../dataflow_etls/orm/__init__.py | 0 .../dataflow_etls/orm/accounts.py | 245 - .../dataflow-etls/dataflow_etls/orm/events.py | 473 -- .../dataflow_etls/transaction_parsing.py | 240 - .../etl/dataflow-etls/dataflow_etls/utils.py | 40 - .../marginfi-v2-account-parsing-batch/job.py | 141 - .../metadata.json | 50 - .../marginfi-v2-account-parsing-stream/job.py | 146 - .../metadata.json | 46 - .../marginfi-v2-event-parsing-batch/job.py | 162 - .../metadata.json | 50 - .../marginfi-v2-event-parsing-stream/job.py | 156 - .../metadata.json | 46 - .../update_running_pipeline | 12 - observability/etl/dataflow-etls/poetry.toml | 3 - .../etl/dataflow-etls/pyproject.toml | 53 - .../dataflow-etls/scripts/build_job_template | 24 - .../scripts/generate_sample_events.sh | 131 - observability/etl/dataflow-etls/scripts/lint | 3 - .../etl/dataflow-etls/scripts/playground.py | 111 - .../dataflow-etls/scripts/sync_job_template | 22 - .../dataflow-etls/scripts/upload_job_template | 16 - observability/etl/dataflow-etls/setup.py | 14 - observability/indexer/build.rs | 18 - observability/indexer/protos/gcp_pubsub.proto | 33 - observability/indexer/protos/geyser.proto | 95 - .../protos/solana-storage-v1.10.40.proto | 118 - .../indexer/src/commands/backfill.rs | 190 - .../indexer/src/commands/index_accounts.rs | 413 -- .../src/commands/index_transactions.rs | 453 -- observability/indexer/src/utils/protos.rs | 3 - 64 files changed, 7869 insertions(+), 19510 deletions(-) create mode 100644 observability/Cargo.lock create mode 100644 observability/Cargo.toml create mode 100644 observability/crates/event_indexer/Cargo.toml create mode 100644 observability/crates/event_indexer/bin/backfill_events.rs create mode 100644 observability/crates/event_indexer/bin/index_events.rs create mode 100644 observability/crates/event_indexer/src/backfiller.rs create mode 100644 observability/crates/event_indexer/src/error.rs create mode 100644 observability/crates/event_indexer/src/indexer.rs create mode 100644 observability/crates/event_indexer/src/lib.rs create mode 100644 observability/crates/event_indexer/src/parser.rs create mode 100644 observability/crates/event_indexer/src/snapshot.rs create mode 100644 observability/crates/rpc_utils/Cargo.toml create mode 100644 observability/crates/rpc_utils/src/lib.rs delete mode 100644 observability/etl/dataflow-etls/.dockerignore delete mode 100644 observability/etl/dataflow-etls/.gcloudignore delete mode 100644 observability/etl/dataflow-etls/.gitignore delete mode 100644 observability/etl/dataflow-etls/.mypy.ini delete mode 100644 observability/etl/dataflow-etls/Dockerfile delete mode 100644 observability/etl/dataflow-etls/MANIFEST.in delete mode 100644 observability/etl/dataflow-etls/README.md delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/__init__.py delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/account_parsing.py delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/idl_versions.py delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/idls/devnet/5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK/marginfi-v0.json delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v0.json delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v1.json delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v2.json delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v3.json delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v4.json delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/idls/mainnet/MFv2hWf31Z9kbCa1snEPYctwafyhdvnV7FZnsebVacA/marginfi-v0.json delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/orm/__init__.py delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/orm/accounts.py delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/orm/events.py delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/transaction_parsing.py delete mode 100644 observability/etl/dataflow-etls/dataflow_etls/utils.py delete mode 100644 observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/job.py delete mode 100644 observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/metadata.json delete mode 100644 observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/job.py delete mode 100644 observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/metadata.json delete mode 100644 observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/job.py delete mode 100644 observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/metadata.json delete mode 100644 observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/job.py delete mode 100644 observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/metadata.json delete mode 100644 observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/update_running_pipeline delete mode 100644 observability/etl/dataflow-etls/poetry.toml delete mode 100644 observability/etl/dataflow-etls/pyproject.toml delete mode 100755 observability/etl/dataflow-etls/scripts/build_job_template delete mode 100755 observability/etl/dataflow-etls/scripts/generate_sample_events.sh delete mode 100755 observability/etl/dataflow-etls/scripts/lint delete mode 100644 observability/etl/dataflow-etls/scripts/playground.py delete mode 100755 observability/etl/dataflow-etls/scripts/sync_job_template delete mode 100755 observability/etl/dataflow-etls/scripts/upload_job_template delete mode 100644 observability/etl/dataflow-etls/setup.py delete mode 100644 observability/indexer/build.rs delete mode 100644 observability/indexer/protos/gcp_pubsub.proto delete mode 100644 observability/indexer/protos/geyser.proto delete mode 100644 observability/indexer/protos/solana-storage-v1.10.40.proto delete mode 100644 observability/indexer/src/commands/backfill.rs delete mode 100644 observability/indexer/src/commands/index_accounts.rs delete mode 100644 observability/indexer/src/commands/index_transactions.rs delete mode 100644 observability/indexer/src/utils/protos.rs diff --git a/.github/workflows/release-program.yaml b/.github/workflows/release-program.yaml index 02090c99..d0f4dbdb 100644 --- a/.github/workflows/release-program.yaml +++ b/.github/workflows/release-program.yaml @@ -59,8 +59,6 @@ jobs: --package marginfi-v2-cli \ --features dev \ -- patch-idl target/idl/${{ env.PROGRAM_LIB_NAME }}.json - rm target/idl/${{ env.PROGRAM_LIB_NAME }}.json - mv target/idl/${{ env.PROGRAM_LIB_NAME }}_patched.json target/idl/${{ env.PROGRAM_LIB_NAME }}.json # Display contents of /target/deploy and /target/idl - run: ls -l target/deploy @@ -100,10 +98,10 @@ jobs: # uses: actions/upload-artifact@v2 # with: # name: ${{ env.PROGRAM_LIB_NAME }}-idl-${{ github.run_id }}-${{ github.run_attempt }} - # path: ./target/idl/${{ env.PROGRAM_LIB_NAME }}_patched.json + # path: ./target/idl/${{ env.PROGRAM_LIB_NAME }}.json # - name: Upload IDL (types) # uses: actions/upload-artifact@v2 # with: # name: ${{ env.PROGRAM_LIB_NAME }}-types-${{ github.run_id }}-${{ github.run_attempt }} - # path: ./target/idl/${{ env.PROGRAM_LIB_NAME }}_patched.ts + # path: ./target/idl/${{ env.PROGRAM_LIB_NAME }}.ts diff --git a/Cargo.lock b/Cargo.lock index 2db1a744..7d4d0260 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -543,28 +543,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-stream" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" -dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.35", - "syn 2.0.55", -] - [[package]] name = "async-trait" version = "0.1.74" @@ -607,15 +585,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "autotools" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8da1805e028a172334c3b680f93e71126f2327622faef2ec3d893c0a4ad77" -dependencies = [ - "cc", -] - [[package]] name = "aws-creds" version = "0.34.1" @@ -642,71 +611,12 @@ dependencies = [ "thiserror", ] -[[package]] -name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - [[package]] name = "az" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" -[[package]] -name = "backoff" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" -dependencies = [ - "futures-core", - "getrandom 0.2.11", - "instant", - "pin-project-lite", - "rand 0.8.5", - "tokio", -] - [[package]] name = "backtrace" version = "0.3.69" @@ -1060,28 +970,6 @@ dependencies = [ "chrono", ] -[[package]] -name = "chrono-tz" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76" -dependencies = [ - "chrono", - "chrono-tz-build", - "phf", -] - -[[package]] -name = "chrono-tz-build" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" -dependencies = [ - "parse-zoneinfo", - "phf", - "phf_codegen", -] - [[package]] name = "cipher" version = "0.3.0" @@ -1248,46 +1136,37 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -1568,12 +1447,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" -[[package]] -name = "dotenv" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" - [[package]] name = "dyn-clone" version = "1.0.16" @@ -1712,26 +1585,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "envconfig" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" -dependencies = [ - "envconfig_derive", -] - -[[package]] -name = "envconfig_derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" -dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.35", - "syn 1.0.109", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -1825,12 +1678,6 @@ dependencies = [ "fixed-macro-impl", ] -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "flate2" version = "1.0.28" @@ -1960,29 +1807,6 @@ dependencies = [ "slab", ] -[[package]] -name = "gcp-bigquery-client" -version = "0.16.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb67dedb3fce3a6d25829158e40b22c5e9aa107fdbae0f2fef564931c9161a6a" -dependencies = [ - "async-stream", - "async-trait", - "dyn-clone", - "hyper", - "hyper-rustls", - "log", - "reqwest", - "serde", - "serde_json", - "thiserror", - "time", - "tokio", - "tokio-stream", - "url", - "yup-oauth2", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -2047,98 +1871,6 @@ dependencies = [ "scroll", ] -[[package]] -name = "google-cloud-auth" -version = "0.9.0" -source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" -dependencies = [ - "async-trait", - "base64 0.21.5", - "google-cloud-metadata", - "google-cloud-token", - "home", - "jsonwebtoken", - "reqwest", - "serde", - "serde_json", - "thiserror", - "time", - "tokio", - "tracing", - "urlencoding", -] - -[[package]] -name = "google-cloud-default" -version = "0.1.0" -source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" -dependencies = [ - "async-trait", - "google-cloud-auth", - "google-cloud-gax", - "google-cloud-pubsub", -] - -[[package]] -name = "google-cloud-gax" -version = "0.13.0" -source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" -dependencies = [ - "google-cloud-token", - "http", - "thiserror", - "tokio", - "tokio-retry", - "tokio-util 0.7.10", - "tonic 0.8.3", - "tower", - "tracing", -] - -[[package]] -name = "google-cloud-googleapis" -version = "0.7.0" -source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" -dependencies = [ - "prost 0.11.9", - "prost-types 0.11.9", - "tonic 0.8.3", -] - -[[package]] -name = "google-cloud-metadata" -version = "0.3.2" -source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" -dependencies = [ - "reqwest", - "thiserror", - "tokio", -] - -[[package]] -name = "google-cloud-pubsub" -version = "0.13.0" -source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" -dependencies = [ - "async-channel", - "async-stream", - "google-cloud-gax", - "google-cloud-googleapis", - "google-cloud-token", - "prost-types 0.11.9", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "google-cloud-token" -version = "0.1.0" -source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" -dependencies = [ - "async-trait", -] - [[package]] name = "h2" version = "0.3.22" @@ -2284,15 +2016,6 @@ dependencies = [ "hmac 0.8.1", ] -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - [[package]] name = "http" version = "0.2.11" @@ -2366,25 +2089,11 @@ dependencies = [ "futures-util", "http", "hyper", - "log", "rustls 0.21.9", - "rustls-native-certs", "tokio", "tokio-rustls 0.24.1", ] -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - [[package]] name = "hyper-tls" version = "0.5.0" @@ -2540,12 +2249,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" - [[package]] name = "jsonrpc-core" version = "18.0.0" @@ -2561,20 +2264,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "jsonwebtoken" -version = "8.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" -dependencies = [ - "base64 0.21.5", - "pem", - "ring 0.16.20", - "serde", - "serde_json", - "simple_asn1", -] - [[package]] name = "keccak" version = "0.1.4" @@ -2820,81 +2509,6 @@ dependencies = [ "type-layout", ] -[[package]] -name = "marginfi-v2-indexer" -version = "0.1.0" -dependencies = [ - "anchor-client", - "anyhow", - "backoff", - "base64 0.21.5", - "bincode", - "bs58 0.4.0", - "bytemuck", - "bytes", - "chrono", - "chrono-tz", - "clap 3.2.25", - "concurrent-queue", - "dotenv", - "envconfig", - "fixed", - "fixed-macro", - "futures", - "gcp-bigquery-client", - "google-cloud-auth", - "google-cloud-default", - "google-cloud-gax", - "google-cloud-googleapis", - "google-cloud-pubsub", - "itertools", - "json", - "lazy_static", - "marginfi", - "prost 0.11.9", - "prost-derive 0.11.9", - "protobuf-src", - "pyth-sdk-solana", - "rayon", - "serde", - "serde_json", - "serde_yaml", - "solana-account-decoder", - "solana-client", - "solana-measure", - "solana-metrics", - "solana-sdk", - "solana-transaction-status", - "spl-token 4.0.0", - "thiserror", - "tokio", - "tokio-stream", - "tonic 0.8.3", - "tonic-build 0.8.4", - "tracing", - "tracing-stackdriver", - "tracing-subscriber", - "uuid", - "yellowstone-grpc-client", - "yellowstone-grpc-proto", - "yup-oauth2", -] - -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "maybe-async" version = "0.2.7" @@ -3028,12 +2642,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - [[package]] name = "native-tls" version = "0.2.11" @@ -3075,16 +2683,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num" version = "0.2.1" @@ -3268,15 +2866,6 @@ dependencies = [ "syn 2.0.55", ] -[[package]] -name = "num_threads" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" -dependencies = [ - "libc", -] - [[package]] name = "number_prefix" version = "0.4.0" @@ -3415,12 +3004,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "parking_lot" version = "0.12.1" @@ -3444,15 +3027,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "parse-zoneinfo" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" -dependencies = [ - "regex", -] - [[package]] name = "paste" version = "1.0.14" @@ -3501,54 +3075,6 @@ dependencies = [ "num", ] -[[package]] -name = "petgraph" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" -dependencies = [ - "fixedbitset", - "indexmap 2.2.6", -] - -[[package]] -name = "phf" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_codegen" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" -dependencies = [ - "phf_generator", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" -dependencies = [ - "phf_shared", - "rand 0.8.5", -] - -[[package]] -name = "phf_shared" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" -dependencies = [ - "siphasher", -] - [[package]] name = "pin-project" version = "1.1.3" @@ -3650,26 +3176,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2 1.0.79", - "syn 1.0.109", -] - -[[package]] -name = "prettyplease" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" -dependencies = [ - "proc-macro2 1.0.79", - "syn 2.0.55", -] - [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -3708,153 +3214,36 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2 1.0.79", - "quote 1.0.35", - "version_check", -] - -[[package]] -name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid 0.1.0", -] - -[[package]] -name = "proc-macro2" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive 0.11.9", -] - -[[package]] -name = "prost" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" -dependencies = [ - "bytes", - "prost-derive 0.12.3", -] - -[[package]] -name = "prost-build" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" -dependencies = [ - "bytes", - "heck 0.4.1", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease 0.1.25", - "prost 0.11.9", - "prost-types 0.11.9", - "regex", - "syn 1.0.109", - "tempfile", - "which", -] - -[[package]] -name = "prost-build" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" -dependencies = [ - "bytes", - "heck 0.4.1", - "itertools", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease 0.2.15", - "prost 0.12.3", - "prost-types 0.12.3", - "regex", - "syn 2.0.55", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2 1.0.79", - "quote 1.0.35", - "syn 1.0.109", -] - -[[package]] -name = "prost-derive" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2 1.0.79", - "quote 1.0.35", - "syn 2.0.55", + "version_check", ] [[package]] -name = "prost-types" -version = "0.11.9" +name = "proc-macro-error-attr" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "prost 0.11.9", + "proc-macro2 1.0.79", + "quote 1.0.35", + "version_check", ] [[package]] -name = "prost-types" -version = "0.12.3" +name = "proc-macro2" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" dependencies = [ - "prost 0.12.3", + "unicode-xid 0.1.0", ] [[package]] -name = "protobuf-src" -version = "1.1.0+21.5" +name = "proc-macro2" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ - "autotools", + "unicode-ident", ] [[package]] @@ -4124,17 +3513,8 @@ checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", - "regex-syntax 0.8.2", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -4145,15 +3525,9 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.8.2" @@ -4515,12 +3889,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "security-framework" version = "2.9.2" @@ -4745,24 +4113,6 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -[[package]] -name = "simple_asn1" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" -dependencies = [ - "num-bigint 0.4.4", - "num-traits", - "thiserror", - "time", -] - -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - [[package]] name = "sized-chunks" version = "0.6.5" @@ -6264,12 +5614,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "synstructure" version = "0.12.6" @@ -6451,8 +5795,6 @@ checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", "itoa", - "libc", - "num_threads", "powerfmt", "serde", "time-core", @@ -6527,16 +5869,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.2.0" @@ -6558,17 +5890,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-retry" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" -dependencies = [ - "pin-project", - "rand 0.8.5", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.23.4" @@ -6699,139 +6020,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tonic" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.13.1", - "bytes", - "flate2", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.11.9", - "prost-derive 0.11.9", - "rustls-native-certs", - "rustls-pemfile", - "tokio", - "tokio-rustls 0.23.4", - "tokio-stream", - "tokio-util 0.7.10", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", - "webpki-roots 0.22.6", -] - -[[package]] -name = "tonic" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.21.5", - "bytes", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.12.3", - "rustls 0.21.9", - "rustls-native-certs", - "rustls-pemfile", - "tokio", - "tokio-rustls 0.24.1", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" -dependencies = [ - "prettyplease 0.1.25", - "proc-macro2 1.0.79", - "prost-build 0.11.9", - "quote 1.0.35", - "syn 1.0.109", -] - -[[package]] -name = "tonic-build" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" -dependencies = [ - "prettyplease 0.2.15", - "proc-macro2 1.0.79", - "prost-build 0.12.3", - "quote 1.0.35", - "syn 2.0.55", -] - -[[package]] -name = "tonic-health" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f80db390246dfb46553481f6024f0082ba00178ea495dbb99e70ba9a4fafb5e1" -dependencies = [ - "async-stream", - "prost 0.12.3", - "tokio", - "tokio-stream", - "tonic 0.10.2", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.5", - "slab", - "tokio", - "tokio-util 0.7.10", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" version = "0.3.2" @@ -6871,27 +6059,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "tracing-opentelemetry" version = "0.17.4" @@ -6905,50 +6072,15 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "tracing-serde" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" -dependencies = [ - "serde", - "tracing-core", -] - -[[package]] -name = "tracing-stackdriver" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff9dd91761e07727176a3dd3a1d64bbb577ea656b7b82fa4be4021832674c49" -dependencies = [ - "Inflector", - "serde", - "serde_json", - "thiserror", - "time", - "tracing-core", - "tracing-subscriber", -] - [[package]] name = "tracing-subscriber" version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex", - "serde", - "serde_json", "sharded-slab", - "smallvec", "thread_local", - "tracing", "tracing-core", - "tracing-log", - "tracing-serde", ] [[package]] @@ -7072,9 +6204,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -7109,27 +6241,12 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf-8" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "uuid" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" -dependencies = [ - "getrandom 0.2.11", -] - [[package]] name = "valuable" version = "0.1.0" @@ -7305,18 +6422,6 @@ version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "winapi" version = "0.3.9" @@ -7631,63 +6736,6 @@ dependencies = [ "time", ] -[[package]] -name = "yellowstone-grpc-client" -version = "1.12.0+solana.1.16.23" -source = "git+https://github.com/rpcpool/yellowstone-grpc.git?rev=a2cd1498ac64baa1017d4a4cdefbf46100215b4c#a2cd1498ac64baa1017d4a4cdefbf46100215b4c" -dependencies = [ - "bytes", - "futures", - "http", - "thiserror", - "tonic 0.10.2", - "tonic-health", - "yellowstone-grpc-proto", -] - -[[package]] -name = "yellowstone-grpc-proto" -version = "1.11.0+solana.1.16.23" -source = "git+https://github.com/rpcpool/yellowstone-grpc.git?rev=a2cd1498ac64baa1017d4a4cdefbf46100215b4c#a2cd1498ac64baa1017d4a4cdefbf46100215b4c" -dependencies = [ - "anyhow", - "bincode", - "prost 0.12.3", - "protobuf-src", - "solana-account-decoder", - "solana-sdk", - "solana-transaction-status", - "tonic 0.10.2", - "tonic-build 0.10.2", -] - -[[package]] -name = "yup-oauth2" -version = "8.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "364ca376b5c04d9b2be9693054e3e0d2d146b363819d0f9a10c6ee66e4c8406b" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.13.1", - "futures", - "http", - "hyper", - "hyper-rustls", - "itertools", - "log", - "percent-encoding", - "rustls 0.21.9", - "rustls-pemfile", - "seahash", - "serde", - "serde_json", - "time", - "tokio", - "tower-service", - "url", -] - [[package]] name = "zerocopy" version = "0.7.26" diff --git a/Cargo.toml b/Cargo.toml index 1752730b..dede58b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["programs/*", "clients/rust/*", "tools/*", "observability/indexer"] +members = ["programs/*", "clients/rust/*", "tools/*"] exclude = ["programs/brick"] [workspace.dependencies] diff --git a/observability/Cargo.lock b/observability/Cargo.lock new file mode 100644 index 00000000..29ef9fd4 --- /dev/null +++ b/observability/Cargo.lock @@ -0,0 +1,6113 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aead" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", + "opaque-debug", +] + +[[package]] +name = "aes-gcm-siv" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589c637f0e68c877bbd59a4599bbe849cac8e5f3e4b5a3ebae8f528cd218dcdc" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "polyval", + "subtle", + "zeroize", +] + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.12", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom 0.2.12", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "anchor-attribute-access-control" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faa5be5b72abea167f87c868379ba3c2be356bfca9e6f474fd055fa0f7eeb4f2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.79", + "quote 1.0.35", + "regex", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-account" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f468970344c7c9f9d03b4da854fd7c54f21305059f53789d0045c1dd803f0018" +dependencies = [ + "anchor-syn", + "anyhow", + "bs58 0.5.0", + "proc-macro2 1.0.79", + "quote 1.0.35", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-constant" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59948e7f9ef8144c2aefb3f32a40c5fce2798baeec765ba038389e82301017ef" +dependencies = [ + "anchor-syn", + "proc-macro2 1.0.79", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-error" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc753c9d1c7981cb8948cf7e162fb0f64558999c0413058e2d43df1df5448086" +dependencies = [ + "anchor-syn", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-event" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38b4e172ba1b52078f53fdc9f11e3dc0668ad27997838a0aad2d148afac8c97" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-program" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eebd21543606ab61e2d83d9da37d24d3886a49f390f9c43a1964735e8c0f0d5" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "anchor-derive-accounts" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4720d899b3686396cced9508f23dab420f1308344456ec78ef76f98fda42af" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "anchor-derive-space" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f495e85480bd96ddeb77b71d499247c7d4e8b501e75ecb234e9ef7ae7bd6552a" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "anchor-lang" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d2d4b20100f1310a774aba3471ef268e5c4ba4d5c28c0bbe663c2658acbc414" +dependencies = [ + "anchor-attribute-access-control", + "anchor-attribute-account", + "anchor-attribute-constant", + "anchor-attribute-error", + "anchor-attribute-event", + "anchor-attribute-program", + "anchor-derive-accounts", + "anchor-derive-space", + "arrayref", + "base64 0.13.1", + "bincode", + "borsh 0.10.3", + "bytemuck", + "getrandom 0.2.12", + "solana-program", + "thiserror", +] + +[[package]] +name = "anchor-spl" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f860599da1c2354e7234c768783049eb42e2f54509ecfc942d2e0076a2da7b" +dependencies = [ + "anchor-lang", + "solana-program", + "spl-associated-token-account 1.1.3", + "spl-token 3.5.0", + "spl-token-2022 0.6.1", +] + +[[package]] +name = "anchor-syn" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a125e4b0cc046cfec58f5aa25038e34cf440151d58f0db3afc55308251fe936d" +dependencies = [ + "anyhow", + "bs58 0.5.0", + "heck 0.3.3", + "proc-macro2 1.0.79", + "quote 1.0.35", + "serde", + "serde_json", + "sha2 0.10.8", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" + +[[package]] +name = "ark-bn254" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a22f4561524cd949590d78d7d4c5df8f592430d221f7f3c9497bbafd8972120f" +dependencies = [ + "ark-ec", + "ark-ff", + "ark-std", +] + +[[package]] +name = "ark-ec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defd9a439d56ac24968cca0571f598a61bc8c55f71d50a89cda591cb750670ba" +dependencies = [ + "ark-ff", + "ark-poly", + "ark-serialize", + "ark-std", + "derivative", + "hashbrown 0.13.2", + "itertools 0.10.5", + "num-traits", + "zeroize", +] + +[[package]] +name = "ark-ff" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec847af850f44ad29048935519032c33da8aa03340876d351dfab5660d2966ba" +dependencies = [ + "ark-ff-asm", + "ark-ff-macros", + "ark-serialize", + "ark-std", + "derivative", + "digest 0.10.7", + "itertools 0.10.5", + "num-bigint 0.4.4", + "num-traits", + "paste", + "rustc_version", + "zeroize", +] + +[[package]] +name = "ark-ff-asm" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" +dependencies = [ + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" +dependencies = [ + "num-bigint 0.4.4", + "num-traits", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "ark-poly" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d320bfc44ee185d899ccbadfa8bc31aab923ce1558716e1997a1e74057fe86bf" +dependencies = [ + "ark-ff", + "ark-serialize", + "ark-std", + "derivative", + "hashbrown 0.13.2", +] + +[[package]] +name = "ark-serialize" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5" +dependencies = [ + "ark-serialize-derive", + "ark-std", + "digest 0.10.7", + "num-bigint 0.4.4", +] + +[[package]] +name = "ark-serialize-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae3281bc6d0fd7e549af32b52511e1302185bd688fd3359fa36423346ff682ea" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "ark-std" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + +[[package]] +name = "array-bytes" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ad284aeb45c13f2fb4f084de4a420ebf447423bdf9386c0540ce33cb3ef4b8c" + +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "ascii" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" + +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-compression" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-mutex" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "async-trait" +version = "0.1.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "autotools" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8da1805e028a172334c3b680f93e71126f2327622faef2ec3d893c0a4ad77" +dependencies = [ + "cc", +] + +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "az" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom 0.2.12", + "instant", + "pin-project-lite", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "blake3" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "digest 0.10.7", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + +[[package]] +name = "borsh" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15bf3650200d8bffa99015595e10f1fbd17de07abbc25bb067da79e769939bfa" +dependencies = [ + "borsh-derive 0.9.3", + "hashbrown 0.11.2", +] + +[[package]] +name = "borsh" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" +dependencies = [ + "borsh-derive 0.10.3", + "hashbrown 0.13.2", +] + +[[package]] +name = "borsh-derive" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6441c552f230375d18e3cc377677914d2ca2b0d36e52129fe15450a2dce46775" +dependencies = [ + "borsh-derive-internal 0.9.3", + "borsh-schema-derive-internal 0.9.3", + "proc-macro-crate 0.1.5", + "proc-macro2 1.0.79", + "syn 1.0.109", +] + +[[package]] +name = "borsh-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" +dependencies = [ + "borsh-derive-internal 0.10.3", + "borsh-schema-derive-internal 0.10.3", + "proc-macro-crate 0.1.5", + "proc-macro2 1.0.79", + "syn 1.0.109", +] + +[[package]] +name = "borsh-derive-internal" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "borsh-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "borsh-schema-derive-internal" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "borsh-schema-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "brotli" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + +[[package]] +name = "bs58" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" + +[[package]] +name = "bv" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8834bb1d8ee5dc048ee3124f2c7c1afcc6bc9aed03f11e9dfd8c69470a5db340" +dependencies = [ + "feature-probe", + "serde", +] + +[[package]] +name = "bytemuck" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "caps" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b" +dependencies = [ + "libc", + "thiserror", +] + +[[package]] +name = "cc" +version = "1.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +dependencies = [ + "jobserver", + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.4", +] + +[[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim 0.10.0", + "termcolor", + "textwrap 0.16.1", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "combine" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +dependencies = [ + "ascii", + "byteorder", + "either", + "memchr", + "unreachable", +] + +[[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "console_log" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89f72f65e8501878b8a004d5a1afb780987e2ce2b4532c562e367a72c57499f" +dependencies = [ + "log", + "web-sys", +] + +[[package]] +name = "const-oid" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3" + +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher", +] + +[[package]] +name = "curve25519-dalek" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" +dependencies = [ + "byteorder", + "digest 0.9.0", + "rand_core 0.5.1", + "serde", + "subtle", + "zeroize", +] + +[[package]] +name = "darling" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2 1.0.79", + "quote 1.0.35", + "strsim 0.10.0", + "syn 2.0.52", +] + +[[package]] +name = "darling_macro" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +dependencies = [ + "darling_core", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "data-encoding" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" + +[[package]] +name = "der" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c" +dependencies = [ + "const-oid", +] + +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint 0.4.4", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derivation-path" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e5c37193a1db1d8ed868c03ec7b152175f26160a5b740e5e484143877e0adf0" + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "dialoguer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59c6f2989294b9a498d3ad5491a79c6deb604617378e1cdc4bfc1c1361fe2f87" +dependencies = [ + "console", + "shell-words", + "tempfile", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "dlopen" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e80ad39f814a9abe68583cd50a2d45c8a67561c3361ab8da240587dda80937" +dependencies = [ + "dlopen_derive", + "lazy_static", + "libc", + "winapi", +] + +[[package]] +name = "dlopen_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f236d9e1b1fbd81cea0f9cbdc8dcc7e8ebcd80e6659cd7cb2ad5f6c05946c581" +dependencies = [ + "libc", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + +[[package]] +name = "dyn-clone" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" + +[[package]] +name = "eager" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe71d579d1812060163dff96056261deb5bf6729b100fa2e36a68b9649ba3d3" + +[[package]] +name = "ed25519" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" +dependencies = [ + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand 0.7.3", + "serde", + "sha2 0.9.9", + "zeroize", +] + +[[package]] +name = "ed25519-dalek-bip32" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d2be62a4061b872c8c0873ee4fc6f101ce7b889d039f019c5fa2af471a59908" +dependencies = [ + "derivation-path", + "ed25519-dalek", + "hmac 0.12.1", + "sha2 0.10.8", +] + +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enum-iterator" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" +dependencies = [ + "enum-iterator-derive", +] + +[[package]] +name = "enum-iterator-derive" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "enum_dispatch" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f33313078bb8d4d05a2733a94ac4c2d8a0df9a2b84424ebf4f33bfc224a890e" +dependencies = [ + "once_cell", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "envconfig" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" +dependencies = [ + "envconfig_derive", +] + +[[package]] +name = "envconfig_derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event_indexer" +version = "0.1.0" +dependencies = [ + "anchor-lang", + "backoff", + "bytemuck", + "crossbeam", + "dotenv", + "envconfig", + "futures", + "marginfi", + "rpc_utils", + "solana-account-decoder", + "solana-client", + "solana-measure", + "solana-rpc-client-api", + "solana-sdk", + "solana-transaction-status", + "spl-token 4.0.0", + "static_assertions", + "thiserror", + "tokio", + "tracing", + "tracing-stackdriver", + "tracing-subscriber", + "yellowstone-grpc-client", + "yellowstone-grpc-proto", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "feature-probe" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da" + +[[package]] +name = "fixed" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4810bcba5534219e7c160473f3a59167e2c98bd7516bc0eec5185848bcd38963" +dependencies = [ + "az", + "bytemuck", + "half", + "typenum", +] + +[[package]] +name = "fixed-macro" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0c48af8cb14e02868f449f8a2187bd78af7a08da201fdc78d518ecb1675bc" +dependencies = [ + "fixed", + "fixed-macro-impl", + "fixed-macro-types", +] + +[[package]] +name = "fixed-macro-impl" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c93086f471c0a1b9c5e300ea92f5cd990ac6d3f8edf27616ef624b8fa6402d4b" +dependencies = [ + "fixed", + "paste", + "proc-macro-error", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "fixed-macro-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "044a61b034a2264a7f65aa0c3cd112a01b4d4ee58baace51fead3f21b993c7e4" +dependencies = [ + "fixed", + "fixed-macro-impl", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "serde", + "typenum", + "version_check", +] + +[[package]] +name = "gethostname" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "goblin" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7666983ed0dd8d21a6f6576ee00053ca0926fb281a5522577a4dbd0f1b54143" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "h2" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.2.5", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash 0.7.8", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash 0.8.11", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] + +[[package]] +name = "histogram" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669" + +[[package]] +name = "hmac" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +dependencies = [ + "crypto-mac", + "digest 0.9.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac-drbg" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ea0a1394df5b6574da6e0c1ade9e78868c9fb0a4e5ef4428e32da4676b85b1" +dependencies = [ + "digest 0.9.0", + "generic-array", + "hmac 0.8.1", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls 0.21.10", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps", + "rand_core 0.6.4", + "rand_xoshiro", + "rayon", + "serde", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", +] + +[[package]] +name = "indicatif" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "jobserver" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "jsonrpc-core" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +dependencies = [ + "futures", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libsecp256k1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9d220bc1feda2ac231cb78c3d26f27676b8cf82c96971f7aeef3d0cf2797c73" +dependencies = [ + "arrayref", + "base64 0.12.3", + "digest 0.9.0", + "hmac-drbg", + "libsecp256k1-core", + "libsecp256k1-gen-ecmult", + "libsecp256k1-gen-genmult", + "rand 0.7.3", + "serde", + "sha2 0.9.9", + "typenum", +] + +[[package]] +name = "libsecp256k1-core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f6ab710cec28cef759c5f18671a27dae2a5f952cdaaee1d8e2908cb2478a80" +dependencies = [ + "crunchy", + "digest 0.9.0", + "subtle", +] + +[[package]] +name = "libsecp256k1-gen-ecmult" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccab96b584d38fac86a83f07e659f0deafd0253dc096dab5a36d53efe653c5c3" +dependencies = [ + "libsecp256k1-core", +] + +[[package]] +name = "libsecp256k1-gen-genmult" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67abfe149395e3aa1c48a2beb32b068e2334402df8181f818d3aee2b304c4f5d" +dependencies = [ + "libsecp256k1-core", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "marginfi" +version = "0.1.0" +dependencies = [ + "anchor-lang", + "anchor-spl", + "bytemuck", + "cfg-if", + "enum_dispatch", + "fixed", + "fixed-macro", + "lazy_static", + "pyth-sdk-solana", + "solana-program", + "solana-security-txt", + "static_assertions", + "switchboard-v2", + "type-layout", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "merlin" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58c38e2799fc0978b65dfff8023ec7843e2330bb462f19198840b34b6582397d" +dependencies = [ + "byteorder", + "keccak", + "rand_core 0.6.4", + "zeroize", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset 0.7.1", + "pin-utils", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" +dependencies = [ + "num-bigint 0.2.6", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-bigint 0.2.6", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive 0.5.11", +] + +[[package]] +name = "num_enum" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a015b430d3c108a207fd776d2e2196aaf8b1cf8cf93253e3a097ff3085076a1" +dependencies = [ + "num_enum_derive 0.6.1", +] + +[[package]] +name = "num_enum" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" +dependencies = [ + "num_enum_derive 0.7.2", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate 1.3.1", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "num_enum_derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" +dependencies = [ + "proc-macro-crate 1.3.1", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" +dependencies = [ + "proc-macro-crate 3.1.0", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "pbkdf2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216eaa586a190f0a738f2f918511eecfa90f13295abec0e457cdebcceda80cbd" +dependencies = [ + "crypto-mac", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "percentage" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd23b938276f14057220b707937bcb42fa76dda7560e57a2da30cb52d557937" +dependencies = [ + "num", +] + +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.2.5", +] + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs8" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cabda3fb821068a9a4fab19a683eac3af12edf0f34b94a8be53c4972b8149d0" +dependencies = [ + "der", + "spki", + "zeroize", +] + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "polyval" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +dependencies = [ + "proc-macro2 1.0.79", + "syn 2.0.52", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit 0.19.15", +] + +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit 0.21.1", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + +[[package]] +name = "proc-macro2" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" +dependencies = [ + "bytes", + "heck 0.4.1", + "itertools 0.11.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.52", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "prost-types" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" +dependencies = [ + "prost", +] + +[[package]] +name = "protobuf-src" +version = "1.1.0+21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" +dependencies = [ + "autotools", +] + +[[package]] +name = "pyth-sdk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7aeef4d5f0a9c98ff5af2ddd84a8b89919c512188305b497a9eb9afa97a949" +dependencies = [ + "borsh 0.10.3", + "borsh-derive 0.10.3", + "getrandom 0.2.12", + "hex", + "schemars", + "serde", +] + +[[package]] +name = "pyth-sdk-solana" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa571ea6ea51102b8fc03303d0e6fea4f788f77bb4e0d65ae2d3c5e384e3187" +dependencies = [ + "borsh 0.10.3", + "borsh-derive 0.10.3", + "bytemuck", + "num-derive 0.3.3", + "num-traits", + "pyth-sdk", + "serde", + "solana-program", + "thiserror", +] + +[[package]] +name = "qstring" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d464fae65fff2680baf48019211ce37aaec0c78e9264c84a3e484717f965104e" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "quinn" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8b432585672228923edbbf64b8b12c14e1112f62e88737655b4a083dbcd78e" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.20.9", + "thiserror", + "tokio", + "tracing", + "webpki", +] + +[[package]] +name = "quinn-proto" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b0b33c13a79f669c85defaf4c275dc86a0c0372807d0ca3d78e0bb87274863" +dependencies = [ + "bytes", + "rand 0.8.5", + "ring 0.16.20", + "rustc-hash", + "rustls 0.20.9", + "rustls-native-certs", + "slab", + "thiserror", + "tinyvec", + "tracing", + "webpki", +] + +[[package]] +name = "quinn-udp" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "641538578b21f5e5c8ea733b736895576d0fe329bb883b937db6f4d163dbaaf4" +dependencies = [ + "libc", + "quinn-proto", + "socket2 0.4.10", + "tracing", + "windows-sys 0.42.0", +] + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2 1.0.79", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.12", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rayon" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rcgen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +dependencies = [ + "pem", + "ring 0.16.20", + "time", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.6", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "reqwest" +version = "0.11.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" +dependencies = [ + "async-compression", + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.10", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-rustls 0.24.1", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.25.4", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.12", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "rpassword" +version = "7.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80472be3c897911d0137b2d2b9055faf6eeac5b14e324073d83bc17b191d7e3f" +dependencies = [ + "libc", + "rtoolbox", + "windows-sys 0.48.0", +] + +[[package]] +name = "rpc_utils" +version = "0.1.0" +dependencies = [ + "backoff", + "futures", + "solana-client", + "solana-sdk", +] + +[[package]] +name = "rtoolbox" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c247d24e63230cdb56463ae328478bd5eac8b8faa8c69461a77e8e323afac90e" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "rust_decimal" +version = "1.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee9164faf726e4f3ece4978b25ca877ddc6802fa77f38cdccb32c7f805ecd70c" +dependencies = [ + "arrayvec", + "num-traits", + "serde", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +dependencies = [ + "log", + "ring 0.16.20", + "sct", + "webpki", +] + +[[package]] +name = "rustls" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "schemars" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scroll" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" + +[[package]] +name = "serde" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "serde_json" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" +dependencies = [ + "darling", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "sha-1" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "sha3" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "keccak", + "opaque-debug", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest 0.10.7", + "keccak", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" + +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "solana-account-decoder" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "815b7eeb8cfc0cc27c3500658845bc0adbfb51a9212814af522f4912e1bdab2e" +dependencies = [ + "Inflector", + "base64 0.21.7", + "bincode", + "bs58 0.4.0", + "bv", + "lazy_static", + "serde", + "serde_derive", + "serde_json", + "solana-address-lookup-table-program", + "solana-config-program", + "solana-sdk", + "spl-token 4.0.0", + "spl-token-2022 0.9.0", + "spl-token-metadata-interface", + "thiserror", + "zstd", +] + +[[package]] +name = "solana-address-lookup-table-program" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9137f2199f70e082f15f91076f31fa6e67d98d40168de759feab12c6b60542a8" +dependencies = [ + "bincode", + "bytemuck", + "log", + "num-derive 0.3.3", + "num-traits", + "rustc_version", + "serde", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-program", + "solana-program-runtime", + "solana-sdk", + "thiserror", +] + +[[package]] +name = "solana-clap-utils" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a99a1aa397ec62a9b10f2f072fb95d78c6f4303bbca11d6af5f3f198e3d62e8" +dependencies = [ + "chrono", + "clap 2.34.0", + "rpassword", + "solana-perf", + "solana-remote-wallet", + "solana-sdk", + "thiserror", + "tiny-bip39", + "uriparse", + "url", +] + +[[package]] +name = "solana-client" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd1d1b50f6937ec5b7b05faa171956dc052ad593d058de5046e325cc0ec4c23" +dependencies = [ + "async-trait", + "bincode", + "futures", + "futures-util", + "indexmap 1.9.3", + "indicatif", + "log", + "quinn", + "rand 0.7.3", + "rayon", + "solana-connection-cache", + "solana-measure", + "solana-metrics", + "solana-pubsub-client", + "solana-quic-client", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-rpc-client-nonce-utils", + "solana-sdk", + "solana-streamer", + "solana-thin-client", + "solana-tpu-client", + "solana-udp-client", + "thiserror", + "tokio", +] + +[[package]] +name = "solana-config-program" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7448528e2fd237e7d7ca93d4d8541a8a9f346b9f947405799d9a6dd5c22aa41c" +dependencies = [ + "bincode", + "chrono", + "serde", + "serde_derive", + "solana-program-runtime", + "solana-sdk", +] + +[[package]] +name = "solana-connection-cache" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51fe3a80fc59a93392a103e6ab492305a6ac614abee70cde6e34fe74fc55dd" +dependencies = [ + "async-trait", + "bincode", + "futures-util", + "indexmap 1.9.3", + "log", + "rand 0.7.3", + "rayon", + "rcgen", + "solana-measure", + "solana-metrics", + "solana-sdk", + "thiserror", + "tokio", +] + +[[package]] +name = "solana-frozen-abi" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8e68a37635d475c40f026bfbc39df3298ce91ec0f4db848979b1dbcd9bc675" +dependencies = [ + "ahash 0.8.11", + "blake3", + "block-buffer 0.10.4", + "bs58 0.4.0", + "bv", + "byteorder", + "cc", + "either", + "generic-array", + "getrandom 0.1.16", + "im", + "lazy_static", + "log", + "memmap2", + "once_cell", + "rand_core 0.6.4", + "rustc_version", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "sha2 0.10.8", + "solana-frozen-abi-macro", + "subtle", + "thiserror", +] + +[[package]] +name = "solana-frozen-abi-macro" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ea45edfe53a4d95f18bd627f1b60e200611a436afd0c58c9c529c085af8965" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "rustc_version", + "syn 2.0.52", +] + +[[package]] +name = "solana-logger" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9db83d89279b0620958ae1278fd52f340c68be79980a5f6ebfb3d4e4623d7241" +dependencies = [ + "env_logger", + "lazy_static", + "log", +] + +[[package]] +name = "solana-measure" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6745b818b9d2d88b0011ac5532e3dcd4cde0bd1613464ee1bcb98db423ab97" +dependencies = [ + "log", + "solana-sdk", +] + +[[package]] +name = "solana-metrics" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5b3782e709a4546a77354e6b0fbc176a34f19b420e65c0d9c9c48f93459fbab" +dependencies = [ + "crossbeam-channel", + "gethostname", + "lazy_static", + "log", + "reqwest", + "solana-sdk", +] + +[[package]] +name = "solana-net-utils" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea302ba1a7186826fecace83da4adce9b288e97ea370999a9aee2bfc71129b" +dependencies = [ + "bincode", + "clap 3.2.25", + "crossbeam-channel", + "log", + "nix", + "rand 0.7.3", + "serde", + "serde_derive", + "socket2 0.4.10", + "solana-logger", + "solana-sdk", + "solana-version", + "tokio", + "url", +] + +[[package]] +name = "solana-perf" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42a5d5de014354c112349667c51f80ce01bca0c6b0bfa027cbc069e972c1c0c7" +dependencies = [ + "ahash 0.8.11", + "bincode", + "bv", + "caps", + "curve25519-dalek", + "dlopen", + "dlopen_derive", + "fnv", + "lazy_static", + "libc", + "log", + "nix", + "rand 0.7.3", + "rayon", + "serde", + "solana-metrics", + "solana-rayon-threadlimit", + "solana-sdk", + "solana-vote-program", +] + +[[package]] +name = "solana-program" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2560d24192b60301c1219c054a34bcd9d9723bb64ec9b5b987882d86c32868e6" +dependencies = [ + "ark-bn254", + "ark-ec", + "ark-ff", + "ark-serialize", + "array-bytes", + "base64 0.21.7", + "bincode", + "bitflags 1.3.2", + "blake3", + "borsh 0.10.3", + "borsh 0.9.3", + "bs58 0.4.0", + "bv", + "bytemuck", + "cc", + "console_error_panic_hook", + "console_log", + "curve25519-dalek", + "getrandom 0.2.12", + "itertools 0.10.5", + "js-sys", + "lazy_static", + "libc", + "libsecp256k1", + "log", + "memoffset 0.9.0", + "num-bigint 0.4.4", + "num-derive 0.3.3", + "num-traits", + "parking_lot", + "rand 0.7.3", + "rand_chacha 0.2.2", + "rustc_version", + "rustversion", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "sha2 0.10.8", + "sha3 0.10.8", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-sdk-macro", + "thiserror", + "tiny-bip39", + "wasm-bindgen", + "zeroize", +] + +[[package]] +name = "solana-program-runtime" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1726697292d3f551898537f921749352e965510a9cfe7e7b2ff7f1a0fcc6e1db" +dependencies = [ + "base64 0.21.7", + "bincode", + "eager", + "enum-iterator", + "itertools 0.10.5", + "libc", + "log", + "num-derive 0.3.3", + "num-traits", + "percentage", + "rand 0.7.3", + "rustc_version", + "serde", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-measure", + "solana-metrics", + "solana-sdk", + "solana_rbpf", + "thiserror", +] + +[[package]] +name = "solana-pubsub-client" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f134152897fe6d3fad3da9945ae452dfc6c2d71465ddce1ad8a423d54ad38bee" +dependencies = [ + "crossbeam-channel", + "futures-util", + "log", + "reqwest", + "semver", + "serde", + "serde_derive", + "serde_json", + "solana-account-decoder", + "solana-rpc-client-api", + "solana-sdk", + "thiserror", + "tokio", + "tokio-stream", + "tokio-tungstenite", + "tungstenite", + "url", +] + +[[package]] +name = "solana-quic-client" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409c0182a32bb11acdf84c96361cff4628e93e7e8b293a8cc43e5ef354ffa46a" +dependencies = [ + "async-mutex", + "async-trait", + "futures", + "itertools 0.10.5", + "lazy_static", + "log", + "quinn", + "quinn-proto", + "quinn-udp", + "rcgen", + "rustls 0.20.9", + "solana-connection-cache", + "solana-measure", + "solana-metrics", + "solana-net-utils", + "solana-rpc-client-api", + "solana-sdk", + "solana-streamer", + "thiserror", + "tokio", +] + +[[package]] +name = "solana-rayon-threadlimit" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce5c2d7f4e92580e6dd18877f0cd5f152e662dbda9c2eed69d29ae9a6f6e5d0" +dependencies = [ + "lazy_static", + "num_cpus", +] + +[[package]] +name = "solana-remote-wallet" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a9e49486e3f31009cfd24869de318e0fac261257f0e87e6f692e0bbf6a053b6" +dependencies = [ + "console", + "dialoguer", + "log", + "num-derive 0.3.3", + "num-traits", + "parking_lot", + "qstring", + "semver", + "solana-sdk", + "thiserror", + "uriparse", +] + +[[package]] +name = "solana-rpc-client" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336cdd2dbb4dcfdb7c905eb45fdd32de30f594b12f00d894160a8e4d12fc76a3" +dependencies = [ + "async-trait", + "base64 0.21.7", + "bincode", + "bs58 0.4.0", + "indicatif", + "log", + "reqwest", + "semver", + "serde", + "serde_derive", + "serde_json", + "solana-account-decoder", + "solana-rpc-client-api", + "solana-sdk", + "solana-transaction-status", + "solana-version", + "solana-vote-program", + "tokio", +] + +[[package]] +name = "solana-rpc-client-api" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0cc64f5092d9c3e0bbfbd459689ffc17617b9f52773ffb7e26a2483a33d5ace" +dependencies = [ + "base64 0.21.7", + "bs58 0.4.0", + "jsonrpc-core", + "reqwest", + "semver", + "serde", + "serde_derive", + "serde_json", + "solana-account-decoder", + "solana-sdk", + "solana-transaction-status", + "solana-version", + "spl-token-2022 0.9.0", + "thiserror", +] + +[[package]] +name = "solana-rpc-client-nonce-utils" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c54440695e2a3b14749b52f2021172aeb2387f8bd95f4e0cc2f97e5d27b5ea4" +dependencies = [ + "clap 2.34.0", + "solana-clap-utils", + "solana-rpc-client", + "solana-sdk", + "thiserror", +] + +[[package]] +name = "solana-sdk" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fc9581f8345a67da71386274084d9a2e35f25689871ad644f5992c786df7c7" +dependencies = [ + "assert_matches", + "base64 0.21.7", + "bincode", + "bitflags 1.3.2", + "borsh 0.10.3", + "bs58 0.4.0", + "bytemuck", + "byteorder", + "chrono", + "derivation-path", + "digest 0.10.7", + "ed25519-dalek", + "ed25519-dalek-bip32", + "generic-array", + "hmac 0.12.1", + "itertools 0.10.5", + "js-sys", + "lazy_static", + "libsecp256k1", + "log", + "memmap2", + "num-derive 0.3.3", + "num-traits", + "num_enum 0.6.1", + "pbkdf2 0.11.0", + "qstring", + "rand 0.7.3", + "rand_chacha 0.2.2", + "rustc_version", + "rustversion", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_with", + "sha2 0.10.8", + "sha3 0.10.8", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-logger", + "solana-program", + "solana-sdk-macro", + "thiserror", + "uriparse", + "wasm-bindgen", +] + +[[package]] +name = "solana-sdk-macro" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d749979b74d6ca1d8b0f1da1d0333332cfac425a34d71ed1149cccc322e0533" +dependencies = [ + "bs58 0.4.0", + "proc-macro2 1.0.79", + "quote 1.0.35", + "rustversion", + "syn 2.0.52", +] + +[[package]] +name = "solana-security-txt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" + +[[package]] +name = "solana-streamer" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a848b9b56af99988e6273ccf79f2bd816633dc3da9ea0eb4488a5b0f8ec820" +dependencies = [ + "async-channel", + "bytes", + "crossbeam-channel", + "futures-util", + "histogram", + "indexmap 1.9.3", + "itertools 0.10.5", + "libc", + "log", + "nix", + "pem", + "percentage", + "pkcs8", + "quinn", + "quinn-proto", + "quinn-udp", + "rand 0.7.3", + "rcgen", + "rustls 0.20.9", + "solana-metrics", + "solana-perf", + "solana-sdk", + "thiserror", + "tokio", + "x509-parser", +] + +[[package]] +name = "solana-thin-client" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db5ba7eddcc0cefc3c5c116387097cb81bb13d7598fbdb3b40c5a964105e879" +dependencies = [ + "bincode", + "log", + "rayon", + "solana-connection-cache", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-sdk", +] + +[[package]] +name = "solana-tpu-client" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bdf56494fd1b509c5428f969a10e4e0865b3eaf40aac1640c8f72dac3112b89" +dependencies = [ + "async-trait", + "bincode", + "futures-util", + "indexmap 1.9.3", + "indicatif", + "log", + "rand 0.7.3", + "rayon", + "solana-connection-cache", + "solana-measure", + "solana-metrics", + "solana-pubsub-client", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-sdk", + "thiserror", + "tokio", +] + +[[package]] +name = "solana-transaction-status" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3c52eaa1977b0121a099243de4b5b44de936e67869d3298400fb6e974a2f7b" +dependencies = [ + "Inflector", + "base64 0.21.7", + "bincode", + "borsh 0.10.3", + "bs58 0.4.0", + "lazy_static", + "log", + "serde", + "serde_derive", + "serde_json", + "solana-account-decoder", + "solana-address-lookup-table-program", + "solana-sdk", + "spl-associated-token-account 2.2.0", + "spl-memo 4.0.0", + "spl-token 4.0.0", + "spl-token-2022 0.9.0", + "thiserror", +] + +[[package]] +name = "solana-udp-client" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1225fb057b8b5e5aa5b0ee01b974e6ef2c6f01727dfd217c23b89b6547a8b17b" +dependencies = [ + "async-trait", + "solana-connection-cache", + "solana-net-utils", + "solana-sdk", + "solana-streamer", + "thiserror", + "tokio", +] + +[[package]] +name = "solana-version" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f7b09ffc8f5446bee6ee1ab4ce4c98504d23222313de1d0ed762f736a3ffe3" +dependencies = [ + "log", + "rustc_version", + "semver", + "serde", + "serde_derive", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-sdk", +] + +[[package]] +name = "solana-vote-program" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2be239ebe1d73af268ce9ba5111ce9595a430aa98576105e87b00e92a5ef2a0b" +dependencies = [ + "bincode", + "log", + "num-derive 0.3.3", + "num-traits", + "rustc_version", + "serde", + "serde_derive", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-metrics", + "solana-program", + "solana-program-runtime", + "solana-sdk", + "thiserror", +] + +[[package]] +name = "solana-zk-token-sdk" +version = "1.16.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4b0547480462cfec9dddaa8adcf2fa7c8b022021738bf71c790c0c7be54a34" +dependencies = [ + "aes-gcm-siv", + "base64 0.21.7", + "bincode", + "bytemuck", + "byteorder", + "curve25519-dalek", + "getrandom 0.1.16", + "itertools 0.10.5", + "lazy_static", + "merlin", + "num-derive 0.3.3", + "num-traits", + "rand 0.7.3", + "serde", + "serde_json", + "sha3 0.9.1", + "solana-program", + "solana-sdk", + "subtle", + "thiserror", + "zeroize", +] + +[[package]] +name = "solana_rbpf" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17d4ba1e58947346e360fabde0697029d36ba83c42f669199b16a8931313cf29" +dependencies = [ + "byteorder", + "combine", + "goblin", + "hash32", + "libc", + "log", + "rand 0.8.5", + "rustc-demangle", + "scroll", + "thiserror", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "spki" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d01ac02a6ccf3e07db148d2be087da624fea0221a16152ed01f0496a6b0a27" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "spl-associated-token-account" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978dba3bcbe88d0c2c58366c254d9ea41c5f73357e72fc0bdee4d6b5fc99c8f4" +dependencies = [ + "assert_matches", + "borsh 0.9.3", + "num-derive 0.3.3", + "num-traits", + "solana-program", + "spl-token 3.5.0", + "spl-token-2022 0.6.1", + "thiserror", +] + +[[package]] +name = "spl-associated-token-account" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385e31c29981488f2820b2022d8e731aae3b02e6e18e2fd854e4c9a94dc44fc3" +dependencies = [ + "assert_matches", + "borsh 0.10.3", + "num-derive 0.4.2", + "num-traits", + "solana-program", + "spl-token 4.0.0", + "spl-token-2022 0.9.0", + "thiserror", +] + +[[package]] +name = "spl-discriminator" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cce5d563b58ef1bb2cdbbfe0dfb9ffdc24903b10ae6a4df2d8f425ece375033f" +dependencies = [ + "bytemuck", + "solana-program", + "spl-discriminator-derive", +] + +[[package]] +name = "spl-discriminator-derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fd7858fc4ff8fb0e34090e41d7eb06a823e1057945c26d480bfc21d2338a93" +dependencies = [ + "quote 1.0.35", + "spl-discriminator-syn", + "syn 2.0.52", +] + +[[package]] +name = "spl-discriminator-syn" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fea7be851bd98d10721782ea958097c03a0c2a07d8d4997041d0ece6319a63" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "sha2 0.10.8", + "syn 2.0.52", + "thiserror", +] + +[[package]] +name = "spl-memo" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0dc6f70db6bacea7ff25870b016a65ba1d1b6013536f08e4fd79a8f9005325" +dependencies = [ + "solana-program", +] + +[[package]] +name = "spl-memo" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f180b03318c3dbab3ef4e1e4d46d5211ae3c780940dd0a28695aba4b59a75a" +dependencies = [ + "solana-program", +] + +[[package]] +name = "spl-pod" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2881dddfca792737c0706fa0175345ab282b1b0879c7d877bad129645737c079" +dependencies = [ + "borsh 0.10.3", + "bytemuck", + "solana-program", + "solana-zk-token-sdk", + "spl-program-error", +] + +[[package]] +name = "spl-program-error" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249e0318493b6bcf27ae9902600566c689b7dfba9f1bdff5893e92253374e78c" +dependencies = [ + "num-derive 0.4.2", + "num-traits", + "solana-program", + "spl-program-error-derive", + "thiserror", +] + +[[package]] +name = "spl-program-error-derive" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1845dfe71fd68f70382232742e758557afe973ae19e6c06807b2c30f5d5cb474" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "sha2 0.10.8", + "syn 2.0.52", +] + +[[package]] +name = "spl-tlv-account-resolution" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "062e148d3eab7b165582757453632ffeef490c02c86a48bfdb4988f63eefb3b9" +dependencies = [ + "bytemuck", + "solana-program", + "spl-discriminator", + "spl-pod", + "spl-program-error", + "spl-type-length-value", +] + +[[package]] +name = "spl-token" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e85e168a785e82564160dcb87b2a8e04cee9bfd1f4d488c729d53d6a4bd300d" +dependencies = [ + "arrayref", + "bytemuck", + "num-derive 0.3.3", + "num-traits", + "num_enum 0.5.11", + "solana-program", + "thiserror", +] + +[[package]] +name = "spl-token" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08459ba1b8f7c1020b4582c4edf0f5c7511a5e099a7a97570c9698d4f2337060" +dependencies = [ + "arrayref", + "bytemuck", + "num-derive 0.3.3", + "num-traits", + "num_enum 0.6.1", + "solana-program", + "thiserror", +] + +[[package]] +name = "spl-token-2022" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0043b590232c400bad5ee9eb983ced003d15163c4c5d56b090ac6d9a57457b47" +dependencies = [ + "arrayref", + "bytemuck", + "num-derive 0.3.3", + "num-traits", + "num_enum 0.5.11", + "solana-program", + "solana-zk-token-sdk", + "spl-memo 3.0.1", + "spl-token 3.5.0", + "thiserror", +] + +[[package]] +name = "spl-token-2022" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4abf34a65ba420584a0c35f3903f8d727d1f13ababbdc3f714c6b065a686e86" +dependencies = [ + "arrayref", + "bytemuck", + "num-derive 0.4.2", + "num-traits", + "num_enum 0.7.2", + "solana-program", + "solana-zk-token-sdk", + "spl-memo 4.0.0", + "spl-pod", + "spl-token 4.0.0", + "spl-token-metadata-interface", + "spl-transfer-hook-interface", + "spl-type-length-value", + "thiserror", +] + +[[package]] +name = "spl-token-metadata-interface" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c16ce3ba6979645fb7627aa1e435576172dd63088dc7848cb09aa331fa1fe4f" +dependencies = [ + "borsh 0.10.3", + "solana-program", + "spl-discriminator", + "spl-pod", + "spl-program-error", + "spl-type-length-value", +] + +[[package]] +name = "spl-transfer-hook-interface" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051d31803f873cabe71aec3c1b849f35248beae5d19a347d93a5c9cccc5d5a9b" +dependencies = [ + "arrayref", + "bytemuck", + "solana-program", + "spl-discriminator", + "spl-pod", + "spl-program-error", + "spl-tlv-account-resolution", + "spl-type-length-value", +] + +[[package]] +name = "spl-type-length-value" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a468e6f6371f9c69aae760186ea9f1a01c2908351b06a5e0026d21cfc4d7ecac" +dependencies = [ + "bytemuck", + "solana-program", + "spl-discriminator", + "spl-pod", + "spl-program-error", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "superslice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab16ced94dbd8a46c82fd81e3ed9a8727dac2977ea869d217bcc4ea1f122e81f" + +[[package]] +name = "switchboard-v2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b81886169f446e22edc18ead7addd9ebd141c39bf2286cb37943c92cd3af724" +dependencies = [ + "anchor-lang", + "anchor-spl", + "bytemuck", + "rust_decimal", + "solana-program", + "superslice", +] + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", + "unicode-xid 0.2.4", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + +[[package]] +name = "thiserror" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-bip39" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc59cb9dfc85bb312c3a78fd6aa8a8582e310b0fa885d5bb877f6dcc601839d" +dependencies = [ + "anyhow", + "hmac 0.8.1", + "once_cell", + "pbkdf2 0.4.0", + "rand 0.7.3", + "rustc-hash", + "sha2 0.9.9", + "thiserror", + "unicode-normalization", + "wasm-bindgen", + "zeroize", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.5.6", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls 0.20.9", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.10", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +dependencies = [ + "futures-util", + "log", + "rustls 0.20.9", + "tokio", + "tokio-rustls 0.23.4", + "tungstenite", + "webpki", + "webpki-roots 0.22.6", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.2.5", + "toml_datetime", + "winnow", +] + +[[package]] +name = "toml_edit" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +dependencies = [ + "indexmap 2.2.5", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "rustls 0.21.10", + "rustls-native-certs", + "rustls-pemfile", + "tokio", + "tokio-rustls 0.24.1", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" +dependencies = [ + "prettyplease", + "proc-macro2 1.0.79", + "prost-build", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "tonic-health" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f80db390246dfb46553481f6024f0082ba00178ea495dbb99e70ba9a4fafb5e1" +dependencies = [ + "async-stream", + "prost", + "tokio", + "tokio-stream", + "tonic", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-stackdriver" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff9dd91761e07727176a3dd3a1d64bbb577ea656b7b82fa4be4021832674c49" +dependencies = [ + "Inflector", + "serde", + "serde_json", + "thiserror", + "time", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +dependencies = [ + "base64 0.13.1", + "byteorder", + "bytes", + "http", + "httparse", + "log", + "rand 0.8.5", + "rustls 0.20.9", + "sha-1", + "thiserror", + "url", + "utf-8", + "webpki", + "webpki-roots 0.22.6", +] + +[[package]] +name = "type-layout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "074069282ba5be5078f1bdb34112b963516d50f262bf4c1082fee1f6ada11d74" +dependencies = [ + "memoffset 0.5.6", + "type-layout-derive", +] + +[[package]] +name = "type-layout-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4a1cf66ce820973c4b31c5ef47a8e930a53ffbcec191212c33f5a3ad75c6cd" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 1.0.109", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "universal-hash" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "uriparse" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0200d0fc04d809396c2ad43f3c95da3582a2556eba8d453c1087f4120ee352ff" +dependencies = [ + "fnv", + "lazy_static", +] + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote 1.0.35", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "x509-parser" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0ecbeb7b67ce215e40e3cc7f2ff902f94a223acf44995934763467e7b1febc8" +dependencies = [ + "asn1-rs", + "base64 0.13.1", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yellowstone-grpc-client" +version = "1.12.0+solana.1.16.23" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?rev=a2cd1498ac64baa1017d4a4cdefbf46100215b4c#a2cd1498ac64baa1017d4a4cdefbf46100215b4c" +dependencies = [ + "bytes", + "futures", + "http", + "thiserror", + "tonic", + "tonic-health", + "yellowstone-grpc-proto", +] + +[[package]] +name = "yellowstone-grpc-proto" +version = "1.11.0+solana.1.16.23" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?rev=a2cd1498ac64baa1017d4a4cdefbf46100215b4c#a2cd1498ac64baa1017d4a4cdefbf46100215b4c" +dependencies = [ + "anyhow", + "bincode", + "prost", + "protobuf-src", + "solana-account-decoder", + "solana-sdk", + "solana-transaction-status", + "tonic", + "tonic-build", +] + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "zeroize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.9+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/observability/Cargo.toml b/observability/Cargo.toml new file mode 100644 index 00000000..2b501722 --- /dev/null +++ b/observability/Cargo.toml @@ -0,0 +1,32 @@ +[workspace] +resolver = "2" +members = ["crates/*"] + +[workspace.dependencies] +anchor-lang = "0.28.0" +backoff = { version = "0.4.0", features = ["tokio"] } +bytemuck = "1.13.1" +concurrent-queue = "2.0.0" +crossbeam = "0.8.2" +dotenv = "0.15.0" +futures = "0.3.25" +envconfig = "0.10.0" +marginfi = { path = "../programs/marginfi", features = [ + "no-entrypoint", + "client", +] } +solana-account-decoder = "1.16.23" +solana-client = "1.16.23" +solana-measure = "1.16.23" +solana-sdk = "1.16.23" +solana-transaction-status = "1.16.23" +solana-rpc-client-api = "1.16.23" +spl-token = "4.0.0" +static_assertions = "1.1.0" +thiserror = "1.0.58" +tokio = { version = "1.14.1", features = ["full"] } +tracing = "0.1.36" +tracing-stackdriver = "0.6.1" +tracing-subscriber = { version = "0.3.15", features = ["env-filter", "fmt"] } +yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", rev = "a2cd1498ac64baa1017d4a4cdefbf46100215b4c" } +yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", rev = "a2cd1498ac64baa1017d4a4cdefbf46100215b4c" } diff --git a/observability/crates/event_indexer/Cargo.toml b/observability/crates/event_indexer/Cargo.toml new file mode 100644 index 00000000..b0af5704 --- /dev/null +++ b/observability/crates/event_indexer/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "event_indexer" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "index-events" +path = "bin/index_events.rs" + +[[bin]] +name = "backfill-events" +path = "bin/backfill_events.rs" + +[features] +default = ["mainnet-beta"] +mainnet-beta = ["marginfi/mainnet-beta"] + +[dependencies] +anchor-lang = { workspace = true } +backoff = { workspace = true } +bytemuck = { workspace = true } +crossbeam = { workspace = true } +dotenv = { workspace = true } +futures = { workspace = true } +envconfig = { workspace = true } +marginfi = { workspace = true } +rpc_utils = { path = "../rpc_utils" } +solana-account-decoder = { workspace = true } +solana-client = { workspace = true } +solana-measure = { workspace = true } +solana-sdk = { workspace = true } +solana-transaction-status = { workspace = true } +solana-rpc-client-api = { workspace = true } +spl-token = { workspace = true } +static_assertions = "1.1.0" +thiserror = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-stackdriver = { workspace = true } +tracing-subscriber = { workspace = true } +yellowstone-grpc-client = { workspace = true } +yellowstone-grpc-proto = { workspace = true } diff --git a/observability/crates/event_indexer/bin/backfill_events.rs b/observability/crates/event_indexer/bin/backfill_events.rs new file mode 100644 index 00000000..566c7ef4 --- /dev/null +++ b/observability/crates/event_indexer/bin/backfill_events.rs @@ -0,0 +1,182 @@ +use std::{panic, process, str::FromStr, thread::available_parallelism}; + +use bytemuck::Contiguous; +use dotenv::dotenv; +use envconfig::Envconfig; +use event_indexer::backfiller::{ + crawl_signatures_for_range, find_boundary_signatures_for_range, TransactionData, + MARGINFI_PROGRAM_GENESIS_SIG, +}; +use solana_client::{ + nonblocking::rpc_client::RpcClient, rpc_client::SerializableTransaction, + rpc_config::RpcBlockConfig, +}; +use solana_sdk::{ + commitment_config::{CommitmentConfig, CommitmentLevel}, + signature::Signature, +}; +use solana_transaction_status::UiTransactionEncoding; +use tracing::info; +use tracing_subscriber::{layer::SubscriberExt, EnvFilter}; + +#[derive(Envconfig, Debug, Clone)] +pub struct Config { + #[envconfig(from = "RPC_HOST")] + pub rpc_host: String, + #[envconfig(from = "RPC_TOKEN")] + pub rpc_token: String, + #[envconfig(from = "BEFORE_SIGNATURE")] + pub before: Option, + #[envconfig(from = "UNTIL_SIGNATURE")] + pub until: Option, + #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] + pub gcp_sa_key: String, + #[envconfig(from = "PRETTY_LOGS")] + pub pretty_logs: Option, +} + +#[tokio::main] +pub async fn main() { + dotenv().ok(); + + let orig_hook = panic::take_hook(); + panic::set_hook(Box::new(move |panic_info| { + orig_hook(panic_info); + process::exit(1); + })); + + let config = Config::init_from_env().unwrap(); + + let pretty_logs = config.pretty_logs.unwrap_or(false); + + let filter = EnvFilter::from_default_env(); + let stackdriver = tracing_stackdriver::layer(); // writes to std::io::Stdout + let subscriber = tracing_subscriber::registry().with(filter); + if pretty_logs { + let subscriber = subscriber.with(tracing_subscriber::fmt::layer().compact()); + tracing::subscriber::set_global_default(subscriber).unwrap(); + } else { + let subscriber = subscriber.with(stackdriver); + tracing::subscriber::set_global_default(subscriber).unwrap(); + }; + + let rpc_endpoint = format!("{}/{}", config.rpc_host, config.rpc_token).to_string(); + let rpc_client = RpcClient::new_with_commitment( + rpc_endpoint.clone(), + CommitmentConfig { + commitment: CommitmentLevel::Confirmed, + }, + ); + + let latest_slot = rpc_client + .get_slot() + .await + .expect("Failed to fetch latest slot"); + let latest_block_slots = rpc_client + .get_blocks(latest_slot - 100, None) + .await + .expect("Failed to fetch block ids"); + if latest_block_slots.is_empty() { + panic!("Failed to find blocks in the last 100 slots"); + } + + let latest_block = rpc_client + .get_block_with_config( + *latest_block_slots.last().unwrap(), + RpcBlockConfig { + encoding: Some(UiTransactionEncoding::Base64), + max_supported_transaction_version: Some(0), + ..Default::default() + }, + ) + .await + .expect("Failed to fetch latest block"); + let before_sig = latest_block + .transactions + .unwrap() + .last() + .unwrap() + .transaction + .decode() + .unwrap() + .get_signature() + .clone(); + + let until_sig = Signature::from_str( + &config + .until + .unwrap_or_else(|| MARGINFI_PROGRAM_GENESIS_SIG.to_string()), + ) + .unwrap(); + + let threads = available_parallelism() + .map(|c| c.into_integer() as usize) + .unwrap_or(1); + + let boundary_sigs = + find_boundary_signatures_for_range(&rpc_client, threads, before_sig, until_sig) + .await + .unwrap(); + + info!("Boundary signatures: {:?}", boundary_sigs); + + let (transaction_tx, transaction_rx) = crossbeam::channel::unbounded::(); + + let mut tasks = vec![]; + for i in 0..threads { + let until_sig = boundary_sigs[i]; + let before_sig = boundary_sigs[i + 1]; + + let local_transaction_tx = transaction_tx.clone(); + let rpc_client = RpcClient::new_with_commitment( + rpc_endpoint.clone(), + CommitmentConfig { + commitment: CommitmentLevel::Confirmed, + }, + ); + + info!( + "Spawning thread for range: {:?}..{:?}", + until_sig, before_sig + ); + + tasks.push(std::thread::spawn(move || { + tokio::runtime::Runtime::new().unwrap().block_on(async { + crawl_signatures_for_range( + i as u64, + rpc_client, + marginfi::ID, + before_sig.1, + until_sig.1, + local_transaction_tx, + None, + ) + .await + }) + })); + } + + let mut tx_count = 0; + while let Ok(TransactionData { + task_id, + transaction, + .. + }) = transaction_rx.recv() + { + tx_count += 1; + let until_slot = boundary_sigs[task_id as usize].0; + let before_slot = boundary_sigs[task_id as usize + 1].0; + println!( + "[{}] {:.1}% complete (total: {}, remaining ranges: {})", + task_id, + (before_slot - transaction.slot) as f64 / (before_slot - until_slot) as f64 * 100.0, + tx_count, + tasks.iter().fold(0, |mut acc, task| { + if !task.is_finished() { + acc = acc + 1; + } + acc + }) + ); + } +} diff --git a/observability/crates/event_indexer/bin/index_events.rs b/observability/crates/event_indexer/bin/index_events.rs new file mode 100644 index 00000000..55bce8a0 --- /dev/null +++ b/observability/crates/event_indexer/bin/index_events.rs @@ -0,0 +1,74 @@ +use dotenv::dotenv; +use envconfig::Envconfig; +use event_indexer::{error::IndexingError, indexer::EventIndexer, snapshot::Snapshot}; +use tracing::{error, info}; +use tracing_subscriber::{layer::SubscriberExt, EnvFilter}; + +use std::{panic, process}; + +#[derive(Envconfig, Debug, Clone)] +pub struct Config { + #[envconfig(from = "RPC_HOST")] + pub rpc_host: String, + #[envconfig(from = "RPC_TOKEN")] + pub rpc_token: String, + #[envconfig(from = "MONITOR_INTERVAL")] + pub monitor_interval: u64, + #[envconfig(from = "PRETTY_LOGS")] + pub pretty_logs: Option, + // #[envconfig(from = "INDEX_TRANSACTIONS_PROJECT_ID")] + // pub project_id: String, + // #[envconfig(from = "INDEX_TRANSACTIONS_PUBSUB_TOPIC_NAME")] + // pub topic_name: String, + // #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] + // pub gcp_sa_key: Option, +} + +#[tokio::main] +pub async fn main() -> Result<(), IndexingError> { + dotenv().ok(); + + let orig_hook = panic::take_hook(); + panic::set_hook(Box::new(move |panic_info| { + orig_hook(panic_info); + process::exit(1); + })); + + let config = Config::init_from_env().unwrap(); + + let pretty_logs = config.pretty_logs.unwrap_or(false); + + let filter = EnvFilter::from_default_env(); + let stackdriver = tracing_stackdriver::layer(); // writes to std::io::Stdout + let subscriber = tracing_subscriber::registry().with(filter); + if pretty_logs { + let subscriber = subscriber.with(tracing_subscriber::fmt::layer().compact()); + tracing::subscriber::set_global_default(subscriber).unwrap(); + } else { + let subscriber = subscriber.with(stackdriver); + tracing::subscriber::set_global_default(subscriber).unwrap(); + }; + + let mut indexer = EventIndexer::new( + config.rpc_host.clone(), + config.rpc_token.clone(), + "".to_string(), + ); + let first_sig = indexer.init().await; + info!("First signature: {:?}", first_sig); + + let rpc_endpoint = format!("{}/{}", config.rpc_host, config.rpc_token).to_string(); + let mut snapshot = Snapshot::new(marginfi::ID, rpc_endpoint); + snapshot.init().await?; + + println!("Snapshot: {}", snapshot); + + let indexer_handle = tokio::spawn(async move { indexer.run().await }); + + match indexer_handle.await { + Ok(_) => info!("Indexer exited successfully"), + Err(e) => error!("Indexer exited with error: {:?}", e), + } + + Ok(()) +} diff --git a/observability/crates/event_indexer/src/backfiller.rs b/observability/crates/event_indexer/src/backfiller.rs new file mode 100644 index 00000000..4620ff0a --- /dev/null +++ b/observability/crates/event_indexer/src/backfiller.rs @@ -0,0 +1,279 @@ +use backoff::{future::retry, ExponentialBackoffBuilder}; +use crossbeam::channel::Sender; +use futures::{future::try_join_all, stream, StreamExt}; +use solana_client::{ + nonblocking::rpc_client::RpcClient, + rpc_client::{GetConfirmedSignaturesForAddress2Config, SerializableTransaction}, + rpc_config::{RpcBlockConfig, RpcTransactionConfig}, + rpc_request::RpcError, +}; +use solana_rpc_client_api::client_error::{Error as ClientError, ErrorKind}; +use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature}; +use solana_transaction_status::{EncodedConfirmedTransactionWithStatusMeta, UiTransactionEncoding}; +use std::{str::FromStr, time::Duration}; +use tracing::{info, warn}; + +use crate::error::IndexingError; + +pub const MARGINFI_PROGRAM_GENESIS_SIG: &str = + "36ViByXbDDgXHo3fTghrrT9zHMu9mYCJMhpKYTpytHcj7Nxkpb6gQdBJRxtDbF53mebNc8HR4aC7pcKmRNypxTWC"; + +#[derive(Debug)] +pub struct TransactionData { + pub task_id: u64, + pub signature: Signature, + pub transaction: EncodedConfirmedTransactionWithStatusMeta, +} + +pub async fn find_boundary_signatures_for_range( + rpc_client: &RpcClient, + task_count: usize, + before: Signature, + until: Signature, +) -> Result, IndexingError> { + let before_slot = rpc_client + .get_transaction_with_config( + &before, + RpcTransactionConfig { + commitment: Some(CommitmentConfig::confirmed()), + max_supported_transaction_version: Some(0), + ..Default::default() + }, + ) + .await + .map_err(|_| IndexingError::FailedToFindTransactionSlot(before))? + .slot; + + let until_slot = rpc_client + .get_transaction_with_config( + &until, + RpcTransactionConfig { + commitment: Some(CommitmentConfig::confirmed()), + max_supported_transaction_version: Some(0), + ..Default::default() + }, + ) + .await + .map_err(|_| IndexingError::FailedToFindTransactionSlot(until))? + .slot; + + let mut boundary_slots = vec![]; + let chunk_length = (before_slot - until_slot) / task_count as u64; + for i in 0..task_count.into() { + let boundary_slot = until_slot + chunk_length * i as u64; + boundary_slots.push(boundary_slot); + } + boundary_slots.push(before_slot); + + let last_boundary_index = boundary_slots.len() - 1; + let boundary_sigs = try_join_all(boundary_slots.into_iter().enumerate().map( + |(index, slot)| async move { + let forward = index != last_boundary_index; + let boundary_slot = find_boundary_signature(rpc_client, slot, forward, None).await?; + Ok((slot, boundary_slot)) + }, + )) + .await?; + + Ok(boundary_sigs) +} + +pub async fn find_boundary_signature( + rpc_client: &RpcClient, + slot: u64, + forward: bool, + max_retries: Option, +) -> Result { + let max_retries = max_retries.unwrap_or(10); + + let mut i = 0; + let mut candidate_slot = slot; + + loop { + let result = rpc_client + .get_block_with_config( + candidate_slot, + RpcBlockConfig { + encoding: Some(UiTransactionEncoding::Base64), + max_supported_transaction_version: Some(0), + ..Default::default() + }, + ) + .await; + + match result { + Ok(block) => { + if let Some(boundary_txs) = block.transactions { + let boundary_tx = if forward { + boundary_txs.last().unwrap() + } else { + boundary_txs.first().unwrap() + }; + let boundary_sig = boundary_tx + .transaction + .decode() + .unwrap() + .get_signature() + .clone(); + + return Ok(boundary_sig); + } else { + warn!("No transactions in block for slot {}", candidate_slot); + } + } + Err(error) => { + match error { + ClientError { + kind: + ErrorKind::RpcError(RpcError::RpcResponseError { + code, + message, + data, + }), + .. + } => { + if code != -32009 { + warn!( + "Error fetching block for slot {}: (code: {}, message: {}, data: {:?}), retrying...", + candidate_slot, code, message, data + ); + continue; + } else { + info!( + "Block for slot {} not available yet, trying next...", + candidate_slot + ) + } + } + _ => { + warn!( + "Error fetching block for slot {}: {:?}, retrying...", + candidate_slot, error + ); + continue; + } + }; + } + } + + i += 1; + if forward { + candidate_slot += i; + } else { + candidate_slot -= i; + } + + if i > max_retries { + return Err(IndexingError::BoundarySignatureNotFound(slot, max_retries)); + } + continue; + } +} + +pub async fn crawl_signatures_for_range( + task_id: u64, + rpc_client: RpcClient, + address: Pubkey, + before: Signature, + until: Signature, + transaction_tx: Sender, + max_concurrent_requests: Option, +) -> Result<(), IndexingError> { + let mut last_fetched_signature = before; + let max_concurrent_requests = max_concurrent_requests.unwrap_or(10); + + loop { + let signatures = retry( + ExponentialBackoffBuilder::new() + .with_max_interval(Duration::from_secs(5)) + .build(), + || async { + match rpc_client + .get_signatures_for_address_with_config( + &address, + GetConfirmedSignaturesForAddress2Config { + before: Some(last_fetched_signature), + until: Some(until), + ..Default::default() + }, + ) + .await + { + Ok(signatures) => Ok(signatures), + Err(e) => Err(backoff::Error::transient(e)), + } + }, + ) + .await + .map_err(|_| IndexingError::FailedToFetchSignatures(until, last_fetched_signature))?; + + if signatures.is_empty() { + break; + } + + last_fetched_signature = + Signature::from_str(&signatures.last().unwrap().signature).unwrap(); + + let successful_signatures = signatures + .into_iter() + .filter_map(|sig_data| match sig_data.err { + Some(_) => None, + None => Some(Signature::from_str(&sig_data.signature).unwrap()), + }) + .collect::>(); + + if successful_signatures.is_empty() { + continue; + } + + stream::iter(successful_signatures) + .map(|signature| { + let rpc_client = &rpc_client; + let transaction_tx_clone = transaction_tx.clone(); + async move { + ( + signature, + transaction_tx_clone, + retry( + ExponentialBackoffBuilder::new() + .with_max_interval(Duration::from_secs(5)) + .build(), + || async { + match rpc_client + .get_transaction_with_config( + &signature, + RpcTransactionConfig { + max_supported_transaction_version: Some(0), + encoding: Some(UiTransactionEncoding::Base64), + ..Default::default() + }, + ) + .await + { + Ok(transaction) => Ok(transaction), + Err(e) => Err(backoff::Error::transient(e)), + } + }, + ) + .await + .unwrap(), + ) + } + }) + .buffered(max_concurrent_requests) + .for_each( + |(signature, transaction_tx_clone, transaction)| async move { + transaction_tx_clone + .send(TransactionData { + task_id, + signature: signature.clone(), + transaction, + }) + .unwrap(); + }, + ) + .await; + } + + Ok(()) +} diff --git a/observability/crates/event_indexer/src/error.rs b/observability/crates/event_indexer/src/error.rs new file mode 100644 index 00000000..6af9eb97 --- /dev/null +++ b/observability/crates/event_indexer/src/error.rs @@ -0,0 +1,20 @@ +use solana_sdk::{pubkey::Pubkey, signature::Signature}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum IndexingError { + #[error("Failed to parse account data for: {0}")] + FailedToParseAccountData(Pubkey), + + #[error("Failed to fetch block for slot {0} after {1} retries")] + BoundarySignatureNotFound(u64, u64), + + #[error("Failed to fetch signatures between {0:?} and {1:?}")] + FailedToFetchSignatures(Signature, Signature), + + #[error("Failed to find slot for tx signature {0:?}")] + FailedToFindTransactionSlot(Signature), + + #[error("An unknown error occurred")] + Unknown, +} diff --git a/observability/crates/event_indexer/src/indexer.rs b/observability/crates/event_indexer/src/indexer.rs new file mode 100644 index 00000000..32c1a218 --- /dev/null +++ b/observability/crates/event_indexer/src/indexer.rs @@ -0,0 +1,269 @@ +use std::{ + collections::{BTreeMap, HashMap}, + thread, + time::{Duration, SystemTime, UNIX_EPOCH}, + vec, +}; + +use crossbeam::channel::{Receiver, Sender}; +use futures::StreamExt; +use solana_client::rpc_client::SerializableTransaction; +use solana_sdk::{pubkey::Pubkey, signature::Signature}; +use solana_transaction_status::{TransactionWithStatusMeta, VersionedTransactionWithStatusMeta}; +use tracing::{error, info, warn}; +use yellowstone_grpc_client::GeyserGrpcClient; +use yellowstone_grpc_proto::{ + convert_from, + geyser::{ + subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequestFilterBlocksMeta, + SubscribeRequestFilterTransactions, + }, +}; + +use super::parser::{MarginfiEventParser, MarginfiEventWithMeta, MARGINFI_GROUP_ADDRESS}; + +const BLOCK_META_BUFFER_LENGTH: usize = 30; + +pub struct EventIndexer { + parser: MarginfiEventParser, + // database_connection: PgConnection, + transaction_rx: Receiver, + event_tx: Sender>, +} + +impl EventIndexer { + pub fn new(rpc_host: String, rpc_auth_token: String, database_connection_url: String) -> Self { + let program_id = marginfi::ID; + + let parser = MarginfiEventParser::new(program_id, MARGINFI_GROUP_ADDRESS); + + let (transaction_tx, transaction_rx) = crossbeam::channel::unbounded::(); + let (event_tx, event_rx) = crossbeam::channel::unbounded::>(); + + tokio::spawn(async move { + listen_to_updates(rpc_host, rpc_auth_token, program_id, transaction_tx).await + }); + + tokio::spawn(async move { store_events(event_rx).await }); + + Self { + parser, + // database_connection: establish_connection(database_connection_url), + transaction_rx, + event_tx, + } + } + + pub async fn init(&mut self) -> Signature { + self.process_first_tx().await + } + + async fn process_first_tx(&mut self) -> Signature { + loop { + while let Ok(TransactionUpdate { + transaction, + slot, + timestamp, + }) = self.transaction_rx.try_recv() + { + let signature = transaction.transaction.get_signature().clone(); + let events = self.parser.extract_events(timestamp, slot, transaction); + self.event_tx.send(events).unwrap(); + return signature; + } + thread::sleep(Duration::from_millis(100)); + } + } + + pub async fn run(&mut self) { + loop { + while let Ok(TransactionUpdate { + slot, + timestamp, + transaction, + }) = self.transaction_rx.try_recv() + { + let events = self.parser.extract_events(timestamp, slot, transaction); + self.event_tx.send(events).unwrap(); + } + + thread::sleep(Duration::from_millis(100)); + } + } +} + +pub struct TransactionUpdate { + pub slot: u64, + pub timestamp: i64, + pub transaction: VersionedTransactionWithStatusMeta, +} + +async fn listen_to_updates( + rpc_host: String, + rpc_auth_token: String, + program_id: Pubkey, + transaction_tx: Sender, +) { + loop { + info!("Connecting geyser client"); + let geyser_client_connection_result = GeyserGrpcClient::connect_with_timeout( + rpc_host.to_owned(), + Some(&rpc_auth_token), + None, + Some(Duration::from_secs(10)), + Some(Duration::from_secs(10)), + false, + ) + .await; + info!("Connected"); + + let mut geyser_client = match geyser_client_connection_result { + Ok(geyser_client) => geyser_client, + Err(err) => { + error!("Error connecting to geyser client: {}", err); + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + }; + + // Subscription config + let blocks_meta_sub = HashMap::from_iter([( + "client".to_string(), + SubscribeRequestFilterBlocksMeta::default(), + )]); + let transactions_sub = HashMap::from_iter([( + "client".to_string(), + SubscribeRequestFilterTransactions { + vote: Some(false), + failed: Some(false), + account_include: vec![program_id.to_string()], + account_exclude: vec![], + ..Default::default() + }, + )]); + let commitment_sub = Some(CommitmentLevel::Confirmed); + + let mut transaction_rx = match geyser_client + .subscribe_once( + HashMap::default(), + HashMap::default(), + transactions_sub, + HashMap::default(), + HashMap::default(), + blocks_meta_sub, + commitment_sub, + vec![], + None, + ) + .await + { + Ok(value) => value, + Err(e) => { + error!("Error subscribing geyser client {e}"); + continue; + } + }; + + let mut tx_buffer: BTreeMap> = BTreeMap::new(); // We use this to avoid having to handling associating a timestamp to a tx in the main loop + let mut latest_blocks: BTreeMap = BTreeMap::new(); + + while let Some(received) = transaction_rx.next().await { + match received { + Ok(received) => { + if let Some(update) = received.update_oneof { + match update { + UpdateOneof::BlockMeta(block_meta) => { + let timestamp = block_meta + .block_time + .map(|unix_timestamp| unix_timestamp.timestamp) + .unwrap_or_else(|| { + warn!( + "No block time found in block_meta, using local clock" + ); + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + as i64 + }); + + latest_blocks.insert(block_meta.slot, (block_meta.slot, timestamp)); + if latest_blocks.len() > BLOCK_META_BUFFER_LENGTH { + latest_blocks.pop_first(); + } + + if let Some(txs) = tx_buffer.remove(&block_meta.slot) { + for tx in txs { + transaction_tx + .send(TransactionUpdate { + slot: block_meta.slot, + timestamp, + transaction: tx, + }) + .unwrap(); + } + } + } + UpdateOneof::Transaction(tx_update) => { + if let Some(tx) = tx_update.transaction { + let transaction_with_meta = + convert_from::create_tx_with_meta(tx).unwrap(); + + let TransactionWithStatusMeta::Complete( + versioned_transaction_with_meta, + ) = transaction_with_meta + else { + error!( + "Discarding tx {:?} because mssing metadata", + transaction_with_meta.transaction_signature() + ); + continue; + }; + + let maybe_block_meta = + latest_blocks.get(&tx_update.slot).copied(); + if let Some((slot, timestamp)) = maybe_block_meta { + transaction_tx + .send(TransactionUpdate { + slot, + timestamp, + transaction: versioned_transaction_with_meta, + }) + .unwrap(); + } else { + tx_buffer + .entry(tx_update.slot) + .or_default() + .push(versioned_transaction_with_meta); + } + } + } + _ => {} + } + } + } + Err(err) => { + error!("Error pulling next update: {}", err); + tokio::time::sleep(Duration::from_secs(1)).await; + break; + } + } + } + + error!("Stream got disconnected"); + } +} + +async fn store_events(event_rx: Receiver>) { + loop { + while let Ok(events) = event_rx.try_recv() { + if !events.is_empty() { + for event in events { + info!("{:?}", event); + } + } + } + + thread::sleep(Duration::from_millis(100)); + } +} diff --git a/observability/crates/event_indexer/src/lib.rs b/observability/crates/event_indexer/src/lib.rs new file mode 100644 index 00000000..2219b7e0 --- /dev/null +++ b/observability/crates/event_indexer/src/lib.rs @@ -0,0 +1,5 @@ +pub mod backfiller; +pub mod error; +pub mod indexer; +pub mod parser; +pub mod snapshot; diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs new file mode 100644 index 00000000..8844b2a2 --- /dev/null +++ b/observability/crates/event_indexer/src/parser.rs @@ -0,0 +1,561 @@ +use std::collections::HashMap; + +use anchor_lang::{AnchorDeserialize, Discriminator}; +use marginfi::{ + instruction::{ + LendingAccountBorrow, LendingAccountCloseBalance, LendingAccountDeposit, + LendingAccountEndFlashloan, LendingAccountLiquidate, LendingAccountRepay, + LendingAccountSettleEmissions, LendingAccountStartFlashloan, LendingAccountWithdraw, + LendingAccountWithdrawEmissions, LendingPoolAccrueBankInterest, LendingPoolAddBank, + LendingPoolAddBankWithSeed, LendingPoolConfigureBank, MarginfiAccountInitialize, + SetNewAccountAuthority, + }, + state::marginfi_group::{BankConfig, BankConfigCompact}, +}; +use solana_sdk::{ + hash::Hash, + instruction::CompiledInstruction, + message::SimpleAddressLoader, + pubkey, + pubkey::Pubkey, + signature::Signature, + transaction::{MessageHash, SanitizedTransaction}, +}; +use solana_transaction_status::{ + InnerInstruction, InnerInstructions, VersionedTransactionWithStatusMeta, +}; +use tracing::{error, warn}; + +const SPL_TRANSFER_DISCRIMINATOR: u8 = 3; +pub const MARGINFI_GROUP_ADDRESS: Pubkey = pubkey!("4qp6Fx6tnZkY5Wropq9wUYgtFxXKwE6viZxFHg3rdAG8"); +const COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT: u64 = 232836972; + +#[derive(Debug)] +pub enum MarginfiEvent { + // User actions + CreateAccount(CreateAccountEvent), + AccountAuthorityTransfer(AccountAuthorityTransferEvent), + Deposit(DepositEvent), + Borrow(BorrowEvent), + Repay(RepayEvent), + Withdraw(WithdrawEvent), + WithdrawEmissions(WithdrawEmissionsEvent), + Liquidate(LiquidateEvent), + + // Admin actions + AddBank(AddBankEvent), +} + +#[derive(Debug)] +pub struct MarginfiEventWithMeta { + pub timestamp: i64, + pub tx_sig: Signature, + pub event: MarginfiEvent, + pub in_flashloan: bool, + pub call_stack: Vec, +} + +#[derive(Debug)] +pub struct CreateAccountEvent { + pub account: Pubkey, + pub authority: Pubkey, +} + +#[derive(Debug)] +pub struct AccountAuthorityTransferEvent { + pub account: Pubkey, + pub old_authority: Pubkey, + pub new_authority: Pubkey, +} + +#[derive(Debug)] +pub struct DepositEvent { + pub account: Pubkey, + pub authority: Pubkey, + pub bank: Pubkey, + pub amount: u64, +} + +#[derive(Debug)] +pub struct BorrowEvent { + pub account: Pubkey, + pub authority: Pubkey, + pub bank: Pubkey, + pub amount: u64, +} + +#[derive(Debug)] +pub struct RepayEvent { + pub account: Pubkey, + pub authority: Pubkey, + pub bank: Pubkey, + pub amount: u64, + pub all: bool, +} + +#[derive(Debug)] +pub struct WithdrawEvent { + pub account: Pubkey, + pub authority: Pubkey, + pub bank: Pubkey, + pub amount: u64, + pub all: bool, +} + +#[derive(Debug)] +pub struct WithdrawEmissionsEvent { + pub account: Pubkey, + pub authority: Pubkey, + pub bank: Pubkey, + pub emissions_mint: Pubkey, + pub amount: u64, +} + +#[derive(Debug)] +pub struct LiquidateEvent { + pub asset_amount: u64, + pub asset_bank: Pubkey, + pub liability_bank: Pubkey, + pub liquidator_account: Pubkey, + pub liquidator_authority: Pubkey, + pub liquidatee_account: Pubkey, +} + +#[derive(Debug)] +pub struct AddBankEvent { + pub bank: Pubkey, + pub mint: Pubkey, + pub config: BankConfig, +} + +pub struct MarginfiEventParser { + program_id: Pubkey, + marginfi_group: Pubkey, +} + +impl MarginfiEventParser { + pub fn new(program_id: Pubkey, marginfi_group: Pubkey) -> Self { + Self { + program_id, + marginfi_group, + } + } + + pub fn extract_events( + &self, + timestamp: i64, + slot: u64, + tx_with_meta: VersionedTransactionWithStatusMeta, + ) -> Vec { + let tx_sig = tx_with_meta.transaction.signatures[0]; + + let mut events: Vec = vec![]; + + let mut in_flashloan = false; + + let sanitized_tx = SanitizedTransaction::try_create( + tx_with_meta.transaction, + MessageHash::Precomputed(Hash::default()), + None, + SimpleAddressLoader::Enabled(tx_with_meta.meta.loaded_addresses), + true, + ) + .unwrap(); + + let mut inner_instructions: HashMap> = HashMap::new(); + for InnerInstructions { + instructions, + index, + } in tx_with_meta + .meta + .inner_instructions + .unwrap_or_default() + .into_iter() + { + inner_instructions.insert(index, instructions); + } + + for (outer_ix_index, instruction) in + sanitized_tx.message().instructions().iter().enumerate() + { + let account_keys = sanitized_tx + .message() + .account_keys() + .iter() + .cloned() + .collect::>(); + let top_level_program_id = instruction.program_id(&account_keys); + + let mut call_stack = vec![]; + + let inner_instructions = inner_instructions + .remove(&(outer_ix_index as u8)) + .unwrap_or_default(); + + if top_level_program_id.eq(&self.program_id) { + // println!("Instruction {}: {:?}", i, top_level_program_id); + let event = self.parse_event( + slot, + &tx_sig, + &instruction, + &inner_instructions, + &account_keys, + &mut in_flashloan, + ); + if let Some(event) = event { + let call_stack = call_stack.iter().cloned().cloned().collect(); + let event_with_meta = MarginfiEventWithMeta { + timestamp, + tx_sig, + event, + in_flashloan, + call_stack, + }; + // info!("Event: {:?}", event_with_meta); + events.push(event_with_meta); + } + } + + if inner_instructions.is_empty() { + continue; + } + + call_stack.push(top_level_program_id); + + for (inner_ix_index, inner_instruction) in inner_instructions.iter().enumerate() { + let cpi_program_id = inner_instruction.instruction.program_id(&account_keys); + + if cpi_program_id.eq(&self.program_id) { + let remaining_instructions = if inner_instructions.len() > inner_ix_index + 1 { + &inner_instructions[(inner_ix_index + 1)..] + } else { + &[] + }; + + let event = self.parse_event( + slot, + &tx_sig, + &inner_instruction.instruction, + remaining_instructions, + &account_keys, + &mut in_flashloan, + ); + if let Some(event) = event { + let call_stack = call_stack.iter().cloned().cloned().collect(); + let event_with_meta = MarginfiEventWithMeta { + timestamp, + tx_sig, + event, + in_flashloan, + call_stack, + }; + // info!("Inner event: {:?}", event_with_meta); + events.push(event_with_meta); + } + } + + if let Some(stack_height) = inner_instruction.stack_height { + if stack_height - 1 > call_stack.len() as u32 { + call_stack.push(cpi_program_id); + } else { + call_stack.truncate(stack_height as usize); + } + } + } + } + + events + } + + pub fn parse_event( + &self, + slot: u64, + tx_signature: &Signature, + instruction: &CompiledInstruction, + remaining_instructions: &[InnerInstruction], + account_keys: &[Pubkey], + in_flashloan: &mut bool, + ) -> Option { + if instruction.data.len() < 8 { + error!("Instruction data too short"); + return None; + } + + let ix_accounts = instruction + .accounts + .iter() + .map(|ix| account_keys[*ix as usize]) + .collect::>(); + + let discriminator: [u8; 8] = instruction.data[..8].try_into().ok()?; + let mut instruction_data = &instruction.data[8..]; + match discriminator { + MarginfiAccountInitialize::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + let marginfi_account = *ix_accounts.get(1).unwrap(); + let authority = *ix_accounts.get(2).unwrap(); + + Some(MarginfiEvent::CreateAccount(CreateAccountEvent { + account: marginfi_account, + authority, + })) + } + SetNewAccountAuthority::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(1).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + let marginfi_account = *ix_accounts.get(0).unwrap(); + let signer = *ix_accounts.get(2).unwrap(); + let new_authority = *ix_accounts.get(3).unwrap(); + + Some(MarginfiEvent::AccountAuthorityTransfer( + AccountAuthorityTransferEvent { + account: marginfi_account, + old_authority: signer, + new_authority, + }, + )) + } + LendingAccountDeposit::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + if remaining_instructions.is_empty() { + warn!( + "Expected non-empty remaining instructions after deposit in {:?}", + tx_signature + ); + return None; + } + + let transfer_ix = &remaining_instructions.get(0).unwrap().instruction; + let spl_transfer_amount = get_spl_transfer_amount(transfer_ix, account_keys)?; + + let marginfi_account = *ix_accounts.get(1).unwrap(); + let signer = *ix_accounts.get(2).unwrap(); + let bank = *ix_accounts.get(3).unwrap(); + + Some(MarginfiEvent::Deposit(DepositEvent { + account: marginfi_account, + authority: signer, + bank, + amount: spl_transfer_amount, + })) + } + LendingAccountBorrow::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + if remaining_instructions.is_empty() { + warn!( + "Expected non-empty remaining instructions after borrow in {:?}", + tx_signature + ); + return None; + } + + let transfer_ix = &remaining_instructions.get(0).unwrap().instruction; + let spl_transfer_amount = get_spl_transfer_amount(transfer_ix, account_keys)?; + + let marginfi_account = *ix_accounts.get(1).unwrap(); + let signer = *ix_accounts.get(2).unwrap(); + let bank = *ix_accounts.get(3).unwrap(); + + Some(MarginfiEvent::Borrow(BorrowEvent { + account: marginfi_account, + authority: signer, + bank, + amount: spl_transfer_amount, + })) + } + LendingAccountRepay::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + let instruction = LendingAccountRepay::deserialize(&mut instruction_data).ok()?; + + if remaining_instructions.is_empty() { + warn!( + "Expected non-empty remaining instructions after repay in {:?}", + tx_signature + ); + return None; + } + + let transfer_ix = &remaining_instructions.get(0).unwrap().instruction; + let spl_transfer_amount = get_spl_transfer_amount(transfer_ix, account_keys)?; + + let marginfi_account = *ix_accounts.get(1).unwrap(); + let signer = *ix_accounts.get(2).unwrap(); + let bank = *ix_accounts.get(3).unwrap(); + + Some(MarginfiEvent::Repay(RepayEvent { + account: marginfi_account, + authority: signer, + bank, + amount: spl_transfer_amount, + all: instruction.repay_all.unwrap_or(false), + })) + } + LendingAccountWithdraw::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + let instruction = + LendingAccountWithdraw::deserialize(&mut instruction_data).ok()?; + + if remaining_instructions.is_empty() { + warn!( + "Expected non-empty remaining instructions after withdraw in {:?}", + tx_signature + ); + return None; + } + + let transfer_ix = &remaining_instructions.get(0).unwrap().instruction; + let spl_transfer_amount = get_spl_transfer_amount(transfer_ix, account_keys)?; + + let marginfi_account = *ix_accounts.get(1).unwrap(); + let signer = *ix_accounts.get(2).unwrap(); + let bank = *ix_accounts.get(3).unwrap(); + + Some(MarginfiEvent::Withdraw(WithdrawEvent { + account: marginfi_account, + authority: signer, + bank, + amount: spl_transfer_amount, + all: instruction.withdraw_all.unwrap_or(false), + })) + } + LendingAccountLiquidate::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + let instruction = + LendingAccountLiquidate::deserialize(&mut instruction_data).ok()?; + + let asset_bank = *ix_accounts.get(1).unwrap(); + let liability_bank = *ix_accounts.get(2).unwrap(); + let liquidator_account = *ix_accounts.get(3).unwrap(); + let liquidator_authority = *ix_accounts.get(4).unwrap(); + let liquidatee_account = *ix_accounts.get(5).unwrap(); + + Some(MarginfiEvent::Liquidate(LiquidateEvent { + asset_amount: instruction.asset_amount, + asset_bank, + liability_bank, + liquidator_account, + liquidator_authority, + liquidatee_account, + })) + } + LendingAccountWithdrawEmissions::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + if remaining_instructions.is_empty() { + return None; + } + + let transfer_ix = &remaining_instructions.get(0).unwrap().instruction; + let spl_transfer_amount = get_spl_transfer_amount(transfer_ix, account_keys)?; + + let marginfi_account = *ix_accounts.get(1).unwrap(); + let signer = *ix_accounts.get(2).unwrap(); + let bank = *ix_accounts.get(3).unwrap(); + let emissions_mint = *ix_accounts.get(4).unwrap(); + + Some(MarginfiEvent::WithdrawEmissions(WithdrawEmissionsEvent { + account: marginfi_account, + authority: signer, + bank, + emissions_mint, + amount: spl_transfer_amount, + })) + } + LendingPoolAddBank::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + let bank_config = if slot < COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT { + BankConfig::deserialize(&mut &instruction_data[..531]).unwrap() + } else { + BankConfigCompact::deserialize(&mut &instruction_data[..531]) + .unwrap() + .into() + }; + + let bank_mint = *ix_accounts.get(3).unwrap(); + let bank = *ix_accounts.get(4).unwrap(); + + Some(MarginfiEvent::AddBank(AddBankEvent { + bank, + mint: bank_mint, + config: bank_config, + })) + } + LendingAccountStartFlashloan::DISCRIMINATOR => { + *in_flashloan = true; + + None + } + LendingAccountEndFlashloan::DISCRIMINATOR => { + *in_flashloan = false; + + None + } + LendingAccountCloseBalance::DISCRIMINATOR + | LendingPoolAccrueBankInterest::DISCRIMINATOR + | LendingAccountSettleEmissions::DISCRIMINATOR + | LendingPoolConfigureBank::DISCRIMINATOR + | LendingPoolAddBankWithSeed::DISCRIMINATOR => None, + _ => { + warn!( + "Unknown instruction discriminator {:?} in {:?}", + discriminator, tx_signature + ); + None + } + } + } +} + +fn get_spl_transfer_amount( + instruction: &CompiledInstruction, + account_keys: &[Pubkey], +) -> Option { + let transfer_ix_pid = instruction.program_id(account_keys); + if !transfer_ix_pid.eq(&spl_token::id()) || instruction.data[0] != SPL_TRANSFER_DISCRIMINATOR { + warn!( + "Expected following instruction to be {:?}/{} in deposit, got {:?}/{:?} instead", + spl_token::id(), + SPL_TRANSFER_DISCRIMINATOR, + transfer_ix_pid, + instruction.data[0] + ); + return None; + } + + let spl_transfer_amount: u64 = u64::from_le_bytes(instruction.data[1..9].try_into().unwrap()); + Some(spl_transfer_amount) +} diff --git a/observability/crates/event_indexer/src/snapshot.rs b/observability/crates/event_indexer/src/snapshot.rs new file mode 100644 index 00000000..df12b33e --- /dev/null +++ b/observability/crates/event_indexer/src/snapshot.rs @@ -0,0 +1,193 @@ +use std::{collections::HashMap, fmt::Display}; + +use anchor_lang::{AccountDeserialize, Discriminator}; +use marginfi::state::{ + marginfi_account::MarginfiAccount, + marginfi_group::{Bank, MarginfiGroup}, +}; +use rpc_utils::{get_multiple_accounts_chunked, AccountWithSlot}; +use solana_account_decoder::{UiAccountEncoding, UiDataSliceConfig}; +use solana_client::{ + nonblocking::rpc_client::RpcClient, + rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig}, +}; +use solana_sdk::{ + account::Account, + commitment_config::{CommitmentConfig, CommitmentLevel}, + pubkey::Pubkey, +}; +use tracing::{info, warn}; + +use crate::error::IndexingError; + +pub struct BankRow { + pub address: Pubkey, + pub mint: Pubkey, + pub mint_decimals: u8, +} + +pub struct UserRow { + pub address: Pubkey, +} + +pub struct AccountRow { + pub address: Pubkey, + pub user: Pubkey, +} + +pub struct Snapshot { + program_id: Pubkey, + rpc_client: RpcClient, + pub banks: HashMap, + pub users: HashMap, + pub accounts: HashMap, +} + +impl Display for Snapshot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Snapshot:\n- Banks: {}\n- Users: {}\n- Accounts: {}", + self.banks.len(), + self.users.len(), + self.accounts.len(), + ) + } +} + +impl Snapshot { + pub fn new(program_id: Pubkey, rpc_endpoint: String) -> Self { + let rpc_client = RpcClient::new_with_commitment( + rpc_endpoint.clone(), + CommitmentConfig { + commitment: CommitmentLevel::Confirmed, + }, + ); + + Self { + program_id, + rpc_client, + banks: HashMap::new(), + users: HashMap::new(), + accounts: HashMap::new(), + } + } + + pub async fn init(&mut self) -> Result<(), IndexingError> { + let config = RpcProgramAccountsConfig { + account_config: RpcAccountInfoConfig { + encoding: Some(UiAccountEncoding::Base64), + data_slice: Some(UiDataSliceConfig { + offset: 0, + length: 0, + }), + ..RpcAccountInfoConfig::default() + }, + ..RpcProgramAccountsConfig::default() + }; + + let start_time = std::time::Instant::now(); + + let all_program_account_keys = self + .rpc_client + .get_program_accounts_with_config(&self.program_id, config) + .await + .unwrap(); + + let elapsed = start_time.elapsed(); + info!( + "Time taken to get {:?} addresses: {:?}", + all_program_account_keys.len(), + elapsed + ); + + let start_time = std::time::Instant::now(); + let all_program_accounts = get_multiple_accounts_chunked( + &self.rpc_client, + &all_program_account_keys + .into_iter() + .map(|(pubkey, _)| pubkey) + .collect::>(), + ) + .await + .unwrap(); + + let elapsed = start_time.elapsed(); + info!( + "Time taken to get {:?} accounts: {:?}", + all_program_accounts.len(), + elapsed + ); + + for ( + pubkey, + AccountWithSlot { + account, + evaluation_slot, + }, + ) in all_program_accounts + { + self.create_entry(&pubkey, evaluation_slot, &account) + .await?; + } + + Ok(()) + } + + // This method assumes that all accounts of interest not owned by the marginfi program are + // inserted in the routing lookup table when a program account is created / received for the + // first time. This is why this only processes program accounts. + pub async fn create_entry( + &mut self, + account_pubkey: &Pubkey, + evaluation_slot: u64, + account: &Account, + ) -> Result<(), IndexingError> { + if account.owner == self.program_id && account.data.len() > 8 { + let discriminator: [u8; 8] = account.data[..8].try_into().unwrap(); + match discriminator { + Bank::DISCRIMINATOR => { + let bank = Bank::try_deserialize(&mut (&account.data as &[u8])) + .map_err(|_| IndexingError::FailedToParseAccountData(*account_pubkey))?; + self.banks.insert( + *account_pubkey, + ( + evaluation_slot, + BankRow { + address: *account_pubkey, + mint: bank.mint, + mint_decimals: bank.mint_decimals, + }, + ), + ); + } + MarginfiAccount::DISCRIMINATOR => { + println!("account data size: {}", account.data.len()); + println!("expected size: {}", std::mem::size_of::()); + + let marginfi_account = + MarginfiAccount::try_deserialize(&mut (&account.data as &[u8])).unwrap(); + self.accounts.insert( + *account_pubkey, + ( + evaluation_slot, + AccountRow { + address: *account_pubkey, + user: marginfi_account.authority, + }, + ), + ); + } + MarginfiGroup::DISCRIMINATOR => {} + _ => { + warn!( + "Unknown account discriminator for account: {:?}", + account_pubkey + ); + } + } + } + + Ok(()) + } +} diff --git a/observability/crates/rpc_utils/Cargo.toml b/observability/crates/rpc_utils/Cargo.toml new file mode 100644 index 00000000..1eff081a --- /dev/null +++ b/observability/crates/rpc_utils/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rpc_utils" +version = "0.1.0" +edition = "2021" + +[dependencies] +backoff = { workspace = true } +futures = { workspace = true } +solana-client = { workspace = true } +solana-sdk = { workspace = true } diff --git a/observability/crates/rpc_utils/src/lib.rs b/observability/crates/rpc_utils/src/lib.rs new file mode 100644 index 00000000..0bab36d9 --- /dev/null +++ b/observability/crates/rpc_utils/src/lib.rs @@ -0,0 +1,58 @@ +use std::{collections::HashMap, iter::zip, time::Duration}; + +use backoff::{future::retry, ExponentialBackoffBuilder}; +use futures::future::try_join_all; +use solana_client::{client_error::ClientError, nonblocking::rpc_client::RpcClient}; +use solana_sdk::{account::Account, commitment_config::CommitmentConfig, pubkey::Pubkey}; + +pub struct AccountWithSlot { + pub evaluation_slot: u64, + pub account: Account, +} + +pub async fn get_multiple_accounts_chunked( + rpc_client: &RpcClient, + keys: &[Pubkey], +) -> Result, ClientError> { + let zips: Result, ClientError> = + try_join_all(keys.chunks(100).map(|pubkey_chunk| async move { + let accounts_with_slot = retry( + ExponentialBackoffBuilder::new() + .with_max_interval(Duration::from_secs(5)) + .build(), + || async { + let response = rpc_client + .get_multiple_accounts_with_commitment( + pubkey_chunk, + CommitmentConfig { + commitment: + solana_sdk::commitment_config::CommitmentLevel::Confirmed, + }, + ) + .await?; + Ok(response + .value + .iter() + .map(|maybe_account| (response.context.slot, maybe_account.clone())) + .collect::>()) + }, + ) + .await?; + Ok(zip(pubkey_chunk, accounts_with_slot)) + })) + .await; + + Ok(HashMap::from_iter(zips?.into_iter().flatten().filter_map( + |(key, (slot, maybe_account))| { + maybe_account.map(|account| { + ( + *key, + AccountWithSlot { + evaluation_slot: slot, + account, + }, + ) + }) + }, + ))) +} diff --git a/observability/etl/dataflow-etls/.dockerignore b/observability/etl/dataflow-etls/.dockerignore deleted file mode 100644 index 7a9b3652..00000000 --- a/observability/etl/dataflow-etls/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -# Ignore everything except for Python files and the requirements file. -* -!setup.py -!MANIFEST.in -!dataflow_etls/ -!jobs/ diff --git a/observability/etl/dataflow-etls/.gcloudignore b/observability/etl/dataflow-etls/.gcloudignore deleted file mode 100644 index 2f71adf0..00000000 --- a/observability/etl/dataflow-etls/.gcloudignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore everything except for Python files and the requirements file. -* -!setup.py -!MANIFEST.in -!Dockerfile -!dataflow_etls/ -!jobs/ diff --git a/observability/etl/dataflow-etls/.gitignore b/observability/etl/dataflow-etls/.gitignore deleted file mode 100644 index 0d2e53e5..00000000 --- a/observability/etl/dataflow-etls/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -__pycache__ -.mypy_cache -.venv -poetry.lock -parsed_event_* -.idea* -beam-temp-* \ No newline at end of file diff --git a/observability/etl/dataflow-etls/.mypy.ini b/observability/etl/dataflow-etls/.mypy.ini deleted file mode 100644 index 8e12ff88..00000000 --- a/observability/etl/dataflow-etls/.mypy.ini +++ /dev/null @@ -1,32 +0,0 @@ -[mypy] -pretty = False -show_absolute_path = True -show_column_numbers = True -show_error_codes = True -files = . -exclude = scripts/playground.py - -# This is mostly equivalent to strict=true as of v0.770 -check_untyped_defs = True -disallow_any_generics = True -disallow_incomplete_defs = True -disallow_subclassing_any = True -disallow_untyped_calls = True -disallow_untyped_decorators = False -disallow_untyped_defs = True -no_implicit_optional = True -no_implicit_reexport = True -strict_equality = True -warn_redundant_casts = True -warn_return_any = True -warn_unused_configs = True -warn_unused_ignores = True - -# It's hard to make tests compliant using unittest.mock -[mypy-tests.*] -check_untyped_defs = False -allow_untyped_defs = True - -# There is no type hinting for pytest -[mypy-pytest] -ignore_missing_imports = True \ No newline at end of file diff --git a/observability/etl/dataflow-etls/Dockerfile b/observability/etl/dataflow-etls/Dockerfile deleted file mode 100644 index 1f8cfe00..00000000 --- a/observability/etl/dataflow-etls/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM gcr.io/dataflow-templates-base/python39-template-launcher-base - -ARG JOB_DIRECTORY -ARG WORKDIR=/dataflow/template -RUN mkdir -p ${WORKDIR} -WORKDIR ${WORKDIR} - -COPY setup.py . -COPY MANIFEST.in . - -ENV FLEX_TEMPLATE_PYTHON_PY_FILE="/${WORKDIR}/job.py" -ENV FLEX_TEMPLATE_PYTHON_SETUP_FILE="/${WORKDIR}/setup.py" - -# Upgrade pip and install the requirements. -RUN pip install --no-cache-dir --upgrade pip \ - && pip install --no-cache-dir . \ - # Download the requirements to speed up launching the Dataflow job. - && pip download --no-cache-dir --dest /tmp/dataflow-etls-requirements-cache . - -COPY dataflow_etls/ dataflow_etls/ -COPY ${JOB_DIRECTORY}/job.py . - -# Since we already downloaded all the dependencies, there's no need to rebuild everything. -ENV PIP_NO_DEPS=True diff --git a/observability/etl/dataflow-etls/MANIFEST.in b/observability/etl/dataflow-etls/MANIFEST.in deleted file mode 100644 index 35ccb062..00000000 --- a/observability/etl/dataflow-etls/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include dataflow_etls/idls/**/*.json \ No newline at end of file diff --git a/observability/etl/dataflow-etls/README.md b/observability/etl/dataflow-etls/README.md deleted file mode 100644 index bf0938b9..00000000 --- a/observability/etl/dataflow-etls/README.md +++ /dev/null @@ -1,26 +0,0 @@ -- Test pipeline locally: - -``` -python job.py \ - --temp_location gs://dataflow_jobs_marginfi_v2/tmp/ \ - --project marginfi-dev \ - --input_table marginfi-dev.marginfi_v2_devnet.transactions_raw \ - --output_table_namespace local_file \ - --cluster devnet \ - --min_idl_version 3 \ - --start_date 2022-11-27 \ - --end_date 2022-11-29 -``` - -- Build image and upload to Artifact Registry: - -``` -./scripts/build_job_template -./scripts/upload_job_template -``` - -- Create/Update template and associate metadata file: - -``` -./scripts/sync_job_template -``` diff --git a/observability/etl/dataflow-etls/dataflow_etls/__init__.py b/observability/etl/dataflow-etls/dataflow_etls/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/observability/etl/dataflow-etls/dataflow_etls/account_parsing.py b/observability/etl/dataflow-etls/dataflow_etls/account_parsing.py deleted file mode 100644 index 8318bd07..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/account_parsing.py +++ /dev/null @@ -1,76 +0,0 @@ -import base64 -from dataclasses import asdict -from datetime import datetime -from typing import List, TypedDict, Dict, Any, Tuple, Generator -from decimal import Decimal - -from anchorpy_core.idl import Idl -from based58 import based58 # type: ignore -from solders.pubkey import Pubkey -import apache_beam as beam # type: ignore -from anchorpy.program.common import NamedInstruction as NamedAccountData - -from dataflow_etls.idl_versions import VersionedProgram, IdlPool, Cluster -from dataflow_etls.orm.accounts import ACCOUNT_UPDATE_TO_RECORD_TYPE, AccountUpdateRecord - -AccountUpdateRaw = TypedDict('AccountUpdateRaw', { - 'id': str, - 'created_at': datetime, - 'timestamp': datetime, - 'owner': str, - 'slot': Decimal, - 'pubkey': str, - 'txn_signature': str, - 'lamports': Decimal, - 'executable': bool, - 'rent_epoch': Decimal, - 'data': str, -}) - - -class OwnerProgramNotSupported(Exception): - pass - - -def parse_account(account_update: AccountUpdateRaw, min_idl_version: int, cluster: Cluster, - idl_pool: IdlPool) -> List[AccountUpdateRecord]: - owner_program_id_str = account_update["owner"] - owner_program_id = Pubkey.from_string(owner_program_id_str) - account_update_slot = int(account_update["slot"]) - - try: - idl_raw, idl_version = idl_pool.get_idl_for_slot(owner_program_id_str, account_update_slot) - except KeyError: - raise OwnerProgramNotSupported(f"Unsupported program {owner_program_id_str}") - - idl = Idl.from_json(idl_raw) - program = VersionedProgram(cluster, idl_version, idl, owner_program_id) - - if idl_version < min_idl_version: - return [] - - account_data_bytes = base64.b64decode(account_update["data"]) - - try: - parsed_account_data: NamedAccountData = program.coder.accounts.parse(account_data_bytes) - except Exception as e: - print(f"failed to parse account data in update {account_update['id']}", e) - return [] - - if parsed_account_data.name not in ACCOUNT_UPDATE_TO_RECORD_TYPE: - print(f"discarding unsupported account type {parsed_account_data.name} in update {account_update['id']}") - return [] - else: - # noinspection PyPep8Naming - AccountUpdateRecordType = ACCOUNT_UPDATE_TO_RECORD_TYPE[parsed_account_data.name] - return [AccountUpdateRecordType(parsed_account_data, account_update, idl_version)] - - -class DispatchEventsDoFn(beam.DoFn): # type: ignore - def process(self, record: AccountUpdateRecord, *args: Tuple[Any], **kwargs: Dict[str, Tuple[Any]]) -> Generator[ - str, None, None]: - yield beam.pvalue.TaggedOutput(record.get_tag(), record) - - -def dictionify_record(record: AccountUpdateRecord) -> Dict[str, Any]: - return asdict(record) diff --git a/observability/etl/dataflow-etls/dataflow_etls/idl_versions.py b/observability/etl/dataflow-etls/dataflow_etls/idl_versions.py deleted file mode 100644 index b1cbbd88..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/idl_versions.py +++ /dev/null @@ -1,94 +0,0 @@ -import glob -import os -from pathlib import Path -from typing import List, Literal, Tuple, Optional, Dict -from anchorpy import Program, Provider, Wallet -from anchorpy.utils.rpc import AsyncClient -from anchorpy_core.idl import Idl -from solders.pubkey import Pubkey - -Cluster = Literal["devnet", "mainnet"] -IdlBoundary = tuple[int, int] -ProgramIdlBoundaries = dict[str, List[IdlBoundary]] -ClusterIdlBoundaries = dict[Cluster, ProgramIdlBoundaries] - - -class VersionedProgram(Program): - version: int - cluster: Cluster - - def __init__(self, cluster: Cluster, version: int, idl: Idl, program_id: Pubkey, - provider: Optional[Provider] = None): - self.version = version - self.cluster = cluster - super(VersionedProgram, self).__init__(idl, program_id, - provider or Provider(AsyncClient("http://localhost:8899"), - Wallet.dummy())) - - -# /!\ Boundaries need to be ordered /!\ -IDL_VERSIONS: ClusterIdlBoundaries = { - "devnet": { - # "A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4": [(196494976, 0), (196520454, 1), (197246719, 2), (197494521, 3)], - "5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK": [], - }, - "mainnet": { - "MFv2hWf31Z9kbCa1snEPYctwafyhdvnV7FZnsebVacA": [], - } -} - - -class ClusterNotSupported(Exception): - pass - - -class IdlPool: - idls_per_program: Dict[str, Tuple[List[Tuple[int, Tuple[int, str]]], str, int]] - - def __init__(self, cluster): - idl_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), f"idls/{cluster}") - try: - boundaries_per_program = IDL_VERSIONS[cluster] - except KeyError: - raise ClusterNotSupported(f"Cluster {cluster} is not supported") - - self.idls_per_program = {} - - for program_id in boundaries_per_program: - # Find latest IDL - sorted_idls = [int(os.path.basename(path).removesuffix(".json").removeprefix("marginfi-v")) for path in - glob.glob(f"{idl_dir}/{program_id}/marginfi-v*.json")] - sorted_idls.sort() - latest_idl_version = sorted_idls[-1] - - path = Path(f"{idl_dir}/{program_id}/marginfi-v{latest_idl_version}.json") - latest_idl_raw = path.read_text() - - self.idls_per_program[program_id] = ([], latest_idl_raw, latest_idl_version) - - # Load all IDLs - boundaries = boundaries_per_program[program_id] - for boundary in boundaries: - version_end_slot = boundary[0] - idl_version = boundary[1] - path = Path(f"{idl_dir}/{program_id}/marginfi-v{idl_version}.json") - idl_raw = path.read_text() - self.idls_per_program[program_id][0].append((version_end_slot, (idl_version, idl_raw))) - - def get_idl_for_slot(self, program_id: str, slot: int) -> Tuple[str, int]: - idl_boundaries, latest_idl, latest_idl_version = self.idls_per_program[program_id] - - idl = None - idl_version = None - for version_end_slot, (current_idl_version, current_idl) in idl_boundaries: - # todo: returns latest for upgrade slot, can throw if tx executed in same slot, before upgrade - if version_end_slot > slot: - idl = current_idl - idl_version = current_idl_version - break - - if idl is None: - idl = latest_idl - idl_version = latest_idl_version - - return idl, idl_version diff --git a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK/marginfi-v0.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK/marginfi-v0.json deleted file mode 100644 index 8229bc32..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK/marginfi-v0.json +++ /dev/null @@ -1,2309 +0,0 @@ -{ - "version": "0.1.0", - "name": "marginfi", - "instructions": [ - { - "name": "marginfiGroupInitialize", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": true - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiGroupConfigure", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - } - ], - "args": [ - { - "name": "config", - "type": { - "defined": "GroupConfig" - } - } - ] - }, - { - "name": "lendingPoolAddBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "bankMint", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": true - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "rent", - "isMut": false, - "isSigner": false - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfig", - "type": { - "defined": "BankConfig" - } - } - ] - }, - { - "name": "lendingPoolConfigureBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfigOpt", - "type": { - "defined": "BankConfigOpt" - } - } - ] - }, - { - "name": "lendingPoolHandleBankruptcy", - "docs": [ - "Handle bad debt of a bankrupt marginfi account for a given bank." - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiAccountInitialize", - "docs": [ - "Initialize a marginfi account for a given group" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": true - }, - { - "name": "authority", - "isMut": false, - "isSigner": true - }, - { - "name": "feePayer", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingAccountDeposit", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountRepay", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "repayAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountWithdraw", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "withdrawAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountBorrow", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountLiquidate", - "docs": [ - "Liquidate a lending account balance of an unhealthy marginfi account" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "assetBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liabBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidatorMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "liquidateeMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankInsuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "assetAmount", - "type": "u64" - } - ] - }, - { - "name": "lendingPoolAccrueBankInterest", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingPoolCollectBankFees", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - } - ], - "accounts": [ - { - "name": "MarginfiAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "group", - "type": "publicKey" - }, - { - "name": "authority", - "type": "publicKey" - }, - { - "name": "lendingAccount", - "type": { - "defined": "LendingAccount" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 64 - ] - } - } - ] - } - }, - { - "name": "MarginfiGroup", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": "publicKey" - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - }, - { - "name": "Bank", - "type": { - "kind": "struct", - "fields": [ - { - "name": "mint", - "type": "publicKey" - }, - { - "name": "mintDecimals", - "type": "u8" - }, - { - "name": "group", - "type": "publicKey" - }, - { - "name": "ignore1", - "type": { - "array": [ - "u8", - 7 - ] - } - }, - { - "name": "assetShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liquidityVault", - "type": "publicKey" - }, - { - "name": "liquidityVaultBump", - "type": "u8" - }, - { - "name": "liquidityVaultAuthorityBump", - "type": "u8" - }, - { - "name": "insuranceVault", - "type": "publicKey" - }, - { - "name": "insuranceVaultBump", - "type": "u8" - }, - { - "name": "insuranceVaultAuthorityBump", - "type": "u8" - }, - { - "name": "ignore2", - "type": { - "array": [ - "u8", - 4 - ] - } - }, - { - "name": "collectedInsuranceFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "feeVault", - "type": "publicKey" - }, - { - "name": "feeVaultBump", - "type": "u8" - }, - { - "name": "feeVaultAuthorityBump", - "type": "u8" - }, - { - "name": "ignore3", - "type": { - "array": [ - "u8", - 6 - ] - } - }, - { - "name": "collectedGroupFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalLiabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalAssetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "lastUpdate", - "type": "i64" - }, - { - "name": "config", - "type": { - "defined": "BankConfig" - } - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - } - ], - "types": [ - { - "name": "GroupEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "AccountEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiAccount", - "type": "publicKey" - }, - { - "name": "marginfiAccountAuthority", - "type": "publicKey" - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "LiquidationBalances", - "type": { - "kind": "struct", - "fields": [ - { - "name": "liquidateeAssetBalance", - "type": "f64" - }, - { - "name": "liquidateeLiabilityBalance", - "type": "f64" - }, - { - "name": "liquidatorAssetBalance", - "type": "f64" - }, - { - "name": "liquidatorLiabilityBalance", - "type": "f64" - } - ] - } - }, - { - "name": "LendingAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "balances", - "type": { - "array": [ - { - "defined": "Balance" - }, - 16 - ] - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 8 - ] - } - } - ] - } - }, - { - "name": "Balance", - "type": { - "kind": "struct", - "fields": [ - { - "name": "active", - "type": "bool" - }, - { - "name": "bankPk", - "type": "publicKey" - }, - { - "name": "ignore1", - "type": { - "array": [ - "u8", - 7 - ] - } - }, - { - "name": "assetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 4 - ] - } - } - ] - } - }, - { - "name": "GroupConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": { - "option": "publicKey" - } - } - ] - } - }, - { - "name": "InterestRateConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "plateauInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "maxInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u128", - 8 - ] - } - } - ] - } - }, - { - "name": "InterestRateConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "plateauInterestRate", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "maxInterestRate", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "insuranceIrFee", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "protocolIrFee", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - } - ] - } - }, - { - "name": "BankConfig", - "docs": [ - "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" - ], - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "assetWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "depositLimit", - "type": "u64" - }, - { - "name": "interestRateConfig", - "type": { - "defined": "InterestRateConfig" - } - }, - { - "name": "operationalState", - "type": { - "defined": "BankOperationalState" - } - }, - { - "name": "oracleSetup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "oracleKeys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - }, - { - "name": "ignore1", - "type": { - "array": [ - "u8", - 6 - ] - } - }, - { - "name": "borrowLimit", - "type": "u64" - }, - { - "name": "riskTier", - "type": { - "defined": "RiskTier" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 6 - ] - } - } - ] - } - }, - { - "name": "WrappedI80F48", - "type": { - "kind": "struct", - "fields": [ - { - "name": "value", - "type": "i128" - } - ] - } - }, - { - "name": "BankConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "assetWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "depositLimit", - "type": { - "option": "u64" - } - }, - { - "name": "borrowLimit", - "type": { - "option": "u64" - } - }, - { - "name": "ignore1", - "type": { - "array": [ - "u64", - 22 - ] - } - }, - { - "name": "operationalState", - "type": { - "option": { - "defined": "BankOperationalState" - } - } - }, - { - "name": "oracle", - "type": { - "option": { - "defined": "OracleConfig" - } - } - }, - { - "name": "interestRateConfig", - "type": { - "option": { - "defined": "InterestRateConfigOpt" - } - } - }, - { - "name": "riskTier", - "type": { - "option": { - "defined": "RiskTier" - } - } - }, - { - "name": "ignore2", - "type": { - "array": [ - "u64", - 21 - ] - } - } - ] - } - }, - { - "name": "OracleConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "setup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "keys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - } - ] - } - }, - { - "name": "BalanceIncreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "RepayOnly" - }, - { - "name": "DepositOnly" - } - ] - } - }, - { - "name": "BalanceDecreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "WithdrawOnly" - }, - { - "name": "BorrowOnly" - }, - { - "name": "BypassBorrowLimit" - } - ] - } - }, - { - "name": "WeightType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BalanceSide", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Assets" - }, - { - "name": "Liabilities" - } - ] - } - }, - { - "name": "RiskRequirementType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BankOperationalState", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Paused" - }, - { - "name": "Operational" - }, - { - "name": "ReduceOnly" - } - ] - } - }, - { - "name": "OracleSetup", - "type": { - "kind": "enum", - "variants": [ - { - "name": "None" - }, - { - "name": "Pyth" - } - ] - } - }, - { - "name": "OracleKey", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Pyth", - "fields": [ - "publicKey" - ] - } - ] - } - }, - { - "name": "RiskTier", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Collateral" - }, - { - "name": "Isolated" - } - ] - } - }, - { - "name": "BankVaultType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Liquidity" - }, - { - "name": "Insurance" - }, - { - "name": "Fee" - } - ] - } - } - ], - "events": [ - { - "name": "MarginfiGroupCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - } - ] - }, - { - "name": "MarginfiGroupConfigureEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "config", - "type": { - "defined": "GroupConfig" - }, - "index": false - } - ] - }, - { - "name": "LendingPoolBankCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - } - ] - }, - { - "name": "LendingPoolBankConfigureEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "config", - "type": { - "defined": "BankConfigOpt" - }, - "index": false - } - ] - }, - { - "name": "LendingPoolBankAccrueInterestEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "delta", - "type": "u64", - "index": false - }, - { - "name": "feesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceCollected", - "type": "f64", - "index": false - } - ] - }, - { - "name": "LendingPoolBankCollectFeesEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "groupFeesCollected", - "type": "f64", - "index": false - }, - { - "name": "groupFeesOutstanding", - "type": "f64", - "index": false - }, - { - "name": "insuranceFeesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceFeesOutstanding", - "type": "f64", - "index": false - } - ] - }, - { - "name": "LendingPoolBankHandleBankruptcyEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "badDebt", - "type": "f64", - "index": false - }, - { - "name": "coveredAmount", - "type": "f64", - "index": false - }, - { - "name": "socializedAmount", - "type": "f64", - "index": false - } - ] - }, - { - "name": "MarginfiAccountCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - } - ] - }, - { - "name": "LendingAccountDepositEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountRepayEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountBorrowEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountWithdrawEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountLiquidateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "liquidateeMarginfiAccount", - "type": "publicKey", - "index": false - }, - { - "name": "liquidateeMarginfiAccountAuthority", - "type": "publicKey", - "index": false - }, - { - "name": "assetBank", - "type": "publicKey", - "index": false - }, - { - "name": "assetMint", - "type": "publicKey", - "index": false - }, - { - "name": "liabilityBank", - "type": "publicKey", - "index": false - }, - { - "name": "liabilityMint", - "type": "publicKey", - "index": false - }, - { - "name": "liquidateePreHealth", - "type": "f64", - "index": false - }, - { - "name": "liquidateePostHealth", - "type": "f64", - "index": false - }, - { - "name": "preBalances", - "type": { - "defined": "LiquidationBalances" - }, - "index": false - }, - { - "name": "postBalances", - "type": { - "defined": "LiquidationBalances" - }, - "index": false - } - ] - } - ], - "errors": [ - { - "code": 6000, - "name": "MathError", - "msg": "Math error" - }, - { - "code": 6001, - "name": "BankNotFound", - "msg": "Invalid bank index" - }, - { - "code": 6002, - "name": "LendingAccountBalanceNotFound", - "msg": "Lending account balance not found" - }, - { - "code": 6003, - "name": "BankAssetCapacityExceeded", - "msg": "Bank deposit capacity exceeded" - }, - { - "code": 6004, - "name": "InvalidTransfer", - "msg": "Invalid transfer" - }, - { - "code": 6005, - "name": "MissingPythOrBankAccount", - "msg": "Missing Pyth or Bank account" - }, - { - "code": 6006, - "name": "MissingPythAccount", - "msg": "Missing Pyth account" - }, - { - "code": 6007, - "name": "InvalidOracleAccount", - "msg": "Invalid Pyth account" - }, - { - "code": 6008, - "name": "MissingBankAccount", - "msg": "Missing Bank account" - }, - { - "code": 6009, - "name": "InvalidBankAccount", - "msg": "Invalid Bank account" - }, - { - "code": 6010, - "name": "BadAccountHealth", - "msg": "Bad account health" - }, - { - "code": 6011, - "name": "LendingAccountBalanceSlotsFull", - "msg": "Lending account balance slots are full" - }, - { - "code": 6012, - "name": "BankAlreadyExists", - "msg": "Bank already exists" - }, - { - "code": 6013, - "name": "IllegalLiquidation", - "msg": "Illegal post liquidation state, account is either not unhealthy or liquidation was too big" - }, - { - "code": 6014, - "name": "AccountNotBankrupt", - "msg": "Account is not bankrupt" - }, - { - "code": 6015, - "name": "BalanceNotBadDebt", - "msg": "Account balance is not bad debt" - }, - { - "code": 6016, - "name": "InvalidConfig", - "msg": "Invalid group config" - }, - { - "code": 6017, - "name": "StaleOracle", - "msg": "Stale oracle data" - }, - { - "code": 6018, - "name": "BankPaused", - "msg": "Bank paused" - }, - { - "code": 6019, - "name": "BankReduceOnly", - "msg": "Bank is ReduceOnly mode" - }, - { - "code": 6020, - "name": "BankAccoutNotFound", - "msg": "Bank is missing" - }, - { - "code": 6021, - "name": "OperationDepositOnly", - "msg": "Operation is deposit-only" - }, - { - "code": 6022, - "name": "OperationWithdrawOnly", - "msg": "Operation is withdraw-only" - }, - { - "code": 6023, - "name": "OperationBorrowOnly", - "msg": "Operation is borrow-only" - }, - { - "code": 6024, - "name": "OperationRepayOnly", - "msg": "Operation is repay-only" - }, - { - "code": 6025, - "name": "NoAssetFound", - "msg": "No asset found" - }, - { - "code": 6026, - "name": "NoLiabilityFound", - "msg": "No liability found" - }, - { - "code": 6027, - "name": "InvalidOracleSetup", - "msg": "Invalid oracle setup" - }, - { - "code": 6028, - "name": "IllegalUtilizationRatio", - "msg": "Invalid bank utilization ratio" - }, - { - "code": 6029, - "name": "BankLiabilityCapacityExceeded", - "msg": "Bank borrow cap exceeded" - }, - { - "code": 6030, - "name": "InvalidPrice", - "msg": "Invalid Price" - }, - { - "code": 6031, - "name": "IsolatedAccountIllegalState", - "msg": "Account can have only one liablity when account is under isolated risk" - } - ] -} diff --git a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v0.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v0.json deleted file mode 100644 index 73470943..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v0.json +++ /dev/null @@ -1,1904 +0,0 @@ -{ - "version": "0.1.0", - "name": "marginfi", - "instructions": [ - { - "name": "marginfiGroupInitialize", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": true - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiGroupConfigure", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - } - ], - "args": [ - { - "name": "config", - "type": { - "defined": "GroupConfig" - } - } - ] - }, - { - "name": "lendingPoolAddBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "bankMint", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": true - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "rent", - "isMut": false, - "isSigner": false - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfig", - "type": { - "defined": "BankConfig" - } - } - ] - }, - { - "name": "lendingPoolConfigureBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfigOpt", - "type": { - "defined": "BankConfigOpt" - } - } - ] - }, - { - "name": "lendingPoolHandleBankruptcy", - "docs": [ - "Handle bad debt of a bankrupt marginfi account for a given bank." - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiAccountInitialize", - "docs": [ - "Initialize a marginfi account for a given group" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": true - }, - { - "name": "authority", - "isMut": false, - "isSigner": true - }, - { - "name": "feePayer", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingAccountDeposit", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountRepay", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "repayAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountWithdraw", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "withdrawAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountBorrow", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountLiquidate", - "docs": [ - "Liquidate a lending account balance of an unhealthy marginfi account" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "assetBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liabBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidatorMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "liquidateeMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankInsuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "assetAmount", - "type": "u64" - } - ] - }, - { - "name": "lendingPoolAccrueBankInterest", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingPoolCollectBankFees", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - } - ], - "accounts": [ - { - "name": "MarginfiAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "group", - "type": "publicKey" - }, - { - "name": "authority", - "type": "publicKey" - }, - { - "name": "lendingAccount", - "type": { - "defined": "LendingAccount" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 64 - ] - } - } - ] - } - }, - { - "name": "MarginfiGroup", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": "publicKey" - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - }, - { - "name": "Bank", - "type": { - "kind": "struct", - "fields": [ - { - "name": "mint", - "type": "publicKey" - }, - { - "name": "mintDecimals", - "type": "u8" - }, - { - "name": "group", - "type": "publicKey" - }, - { - "name": "assetShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liquidityVault", - "type": "publicKey" - }, - { - "name": "liquidityVaultBump", - "type": "u8" - }, - { - "name": "liquidityVaultAuthorityBump", - "type": "u8" - }, - { - "name": "insuranceVault", - "type": "publicKey" - }, - { - "name": "insuranceVaultBump", - "type": "u8" - }, - { - "name": "insuranceVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedInsuranceFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "feeVault", - "type": "publicKey" - }, - { - "name": "feeVaultBump", - "type": "u8" - }, - { - "name": "feeVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedGroupFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalLiabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalAssetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "lastUpdate", - "type": "i64" - }, - { - "name": "config", - "type": { - "defined": "BankConfig" - } - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - } - ], - "types": [ - { - "name": "GroupEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "version", - "type": "string" - }, - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "AccountEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "version", - "type": "string" - }, - { - "name": "signer", - "type": "publicKey" - }, - { - "name": "marginfiAccount", - "type": "publicKey" - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "LendingAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "balances", - "type": { - "array": [ - { - "defined": "Balance" - }, - 16 - ] - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 8 - ] - } - } - ] - } - }, - { - "name": "Balance", - "type": { - "kind": "struct", - "fields": [ - { - "name": "active", - "type": "bool" - }, - { - "name": "bankPk", - "type": "publicKey" - }, - { - "name": "assetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 4 - ] - } - } - ] - } - }, - { - "name": "GroupConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": { - "option": "publicKey" - } - } - ] - } - }, - { - "name": "InterestRateConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "plateauInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "maxInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u128", - 8 - ] - } - } - ] - } - }, - { - "name": "BankConfig", - "docs": [ - "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" - ], - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "assetWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "depositLimit", - "type": "u64" - }, - { - "name": "interestRateConfig", - "type": { - "defined": "InterestRateConfig" - } - }, - { - "name": "operationalState", - "type": { - "defined": "BankOperationalState" - } - }, - { - "name": "oracleSetup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "oracleKeys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - }, - { - "name": "borrowLimit", - "type": "u64" - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 7 - ] - } - } - ] - } - }, - { - "name": "WrappedI80F48", - "type": { - "kind": "struct", - "fields": [ - { - "name": "value", - "type": "i128" - } - ] - } - }, - { - "name": "BankConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "assetWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "depositLimit", - "type": { - "option": "u64" - } - }, - { - "name": "borrowLimit", - "type": { - "option": "u64" - } - }, - { - "name": "operationalState", - "type": { - "option": { - "defined": "BankOperationalState" - } - } - }, - { - "name": "oracle", - "type": { - "option": { - "defined": "OracleConfig" - } - } - } - ] - } - }, - { - "name": "OracleConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "setup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "keys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - } - ] - } - }, - { - "name": "BalanceIncreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "RepayOnly" - }, - { - "name": "DepositOnly" - } - ] - } - }, - { - "name": "BalanceDecreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "WithdrawOnly" - }, - { - "name": "BorrowOnly" - }, - { - "name": "BypassBorrowLimit" - } - ] - } - }, - { - "name": "WeightType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BalanceSide", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Assets" - }, - { - "name": "Liabilities" - } - ] - } - }, - { - "name": "RiskRequirementType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BankOperationalState", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Paused" - }, - { - "name": "Operational" - }, - { - "name": "ReduceOnly" - } - ] - } - }, - { - "name": "OracleSetup", - "type": { - "kind": "enum", - "variants": [ - { - "name": "None" - }, - { - "name": "Pyth" - } - ] - } - }, - { - "name": "OracleKey", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Pyth", - "fields": [ - "publicKey" - ] - } - ] - } - }, - { - "name": "BankVaultType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Liquidity" - }, - { - "name": "Insurance" - }, - { - "name": "Fee" - } - ] - } - } - ], - "events": [ - { - "name": "LendingPoolBankAddEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - } - ] - }, - { - "name": "LendingPoolBankAccrueInterestEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "delta", - "type": "u64", - "index": false - }, - { - "name": "feesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceCollected", - "type": "f64", - "index": false - } - ] - }, - { - "name": "MarginfiAccountCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - } - ] - }, - { - "name": "LendingAccountDepositEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountRepayEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountBorrowEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountWithdrawEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - } - ], - "errors": [ - { - "code": 6000, - "name": "MathError", - "msg": "Math error" - }, - { - "code": 6001, - "name": "BankNotFound", - "msg": "Invalid bank index" - }, - { - "code": 6002, - "name": "LendingAccountBalanceNotFound", - "msg": "Lending account balance not found" - }, - { - "code": 6003, - "name": "BankAssetCapacityExceeded", - "msg": "Bank deposit capacity exceeded" - }, - { - "code": 6004, - "name": "InvalidTransfer", - "msg": "Invalid transfer" - }, - { - "code": 6005, - "name": "MissingPythOrBankAccount", - "msg": "Missing Pyth or Bank account" - }, - { - "code": 6006, - "name": "MissingPythAccount", - "msg": "Missing Pyth account" - }, - { - "code": 6007, - "name": "InvalidOracleAccount", - "msg": "Invalid Pyth account" - }, - { - "code": 6008, - "name": "MissingBankAccount", - "msg": "Missing Bank account" - }, - { - "code": 6009, - "name": "InvalidBankAccount", - "msg": "Invalid Bank account" - }, - { - "code": 6010, - "name": "BadAccountHealth", - "msg": "Bad account health" - }, - { - "code": 6011, - "name": "LendingAccountBalanceSlotsFull", - "msg": "Lending account balance slots are full" - }, - { - "code": 6012, - "name": "BankAlreadyExists", - "msg": "Bank already exists" - }, - { - "code": 6013, - "name": "IllegalLiquidation", - "msg": "Illegal post liquidation state, account is either not unhealthy or liquidation was too big" - }, - { - "code": 6014, - "name": "AccountNotBankrupt", - "msg": "Account is not bankrupt" - }, - { - "code": 6015, - "name": "BalanceNotBadDebt", - "msg": "Account balance is not bad debt" - }, - { - "code": 6016, - "name": "InvalidConfig", - "msg": "Invalid group config" - }, - { - "code": 6017, - "name": "StaleOracle", - "msg": "Stale oracle data" - }, - { - "code": 6018, - "name": "BankPaused", - "msg": "Bank paused" - }, - { - "code": 6019, - "name": "BankReduceOnly", - "msg": "Bank is ReduceOnly mode" - }, - { - "code": 6020, - "name": "BankAccoutNotFound", - "msg": "Bank is missing" - }, - { - "code": 6021, - "name": "OperationDepositOnly", - "msg": "Operation is deposit-only" - }, - { - "code": 6022, - "name": "OperationWithdrawOnly", - "msg": "Operation is withdraw-only" - }, - { - "code": 6023, - "name": "OperationBorrowOnly", - "msg": "Operation is borrow-only" - }, - { - "code": 6024, - "name": "OperationRepayOnly", - "msg": "Operation is repay-only" - }, - { - "code": 6025, - "name": "NoAssetFound", - "msg": "No asset found" - }, - { - "code": 6026, - "name": "NoLiabilityFound", - "msg": "No liability found" - }, - { - "code": 6027, - "name": "InvalidOracleSetup", - "msg": "Invalid oracle setup" - }, - { - "code": 6028, - "name": "IllegalUtilizationRatio", - "msg": "Invalid bank utilization ratio" - }, - { - "code": 6029, - "name": "BankLiabilityCapacityExceeded", - "msg": "Bank borrow cap exceeded" - } - ] -} \ No newline at end of file diff --git a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v1.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v1.json deleted file mode 100644 index 2d27f7a8..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v1.json +++ /dev/null @@ -1,1907 +0,0 @@ -{ - "version": "0.1.0", - "name": "marginfi", - "instructions": [ - { - "name": "marginfiGroupInitialize", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": true - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiGroupConfigure", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - } - ], - "args": [ - { - "name": "config", - "type": { - "defined": "GroupConfig" - } - } - ] - }, - { - "name": "lendingPoolAddBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "bankMint", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": true - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "rent", - "isMut": false, - "isSigner": false - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfig", - "type": { - "defined": "BankConfig" - } - } - ] - }, - { - "name": "lendingPoolConfigureBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfigOpt", - "type": { - "defined": "BankConfigOpt" - } - } - ] - }, - { - "name": "lendingPoolHandleBankruptcy", - "docs": [ - "Handle bad debt of a bankrupt marginfi account for a given bank." - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiAccountInitialize", - "docs": [ - "Initialize a marginfi account for a given group" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": true - }, - { - "name": "authority", - "isMut": false, - "isSigner": true - }, - { - "name": "feePayer", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingAccountDeposit", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountRepay", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "repayAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountWithdraw", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "withdrawAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountBorrow", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountLiquidate", - "docs": [ - "Liquidate a lending account balance of an unhealthy marginfi account" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "assetBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liabBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidatorMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "liquidateeMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankInsuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "assetAmount", - "type": "u64" - } - ] - }, - { - "name": "lendingPoolAccrueBankInterest", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingPoolCollectBankFees", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - } - ], - "accounts": [ - { - "name": "MarginfiAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "group", - "type": "publicKey" - }, - { - "name": "authority", - "type": "publicKey" - }, - { - "name": "lendingAccount", - "type": { - "defined": "LendingAccount" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 64 - ] - } - } - ] - } - }, - { - "name": "MarginfiGroup", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": "publicKey" - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - }, - { - "name": "Bank", - "type": { - "kind": "struct", - "fields": [ - { - "name": "mint", - "type": "publicKey" - }, - { - "name": "mintDecimals", - "type": "u8" - }, - { - "name": "group", - "type": "publicKey" - }, - { - "name": "assetShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liquidityVault", - "type": "publicKey" - }, - { - "name": "liquidityVaultBump", - "type": "u8" - }, - { - "name": "liquidityVaultAuthorityBump", - "type": "u8" - }, - { - "name": "insuranceVault", - "type": "publicKey" - }, - { - "name": "insuranceVaultBump", - "type": "u8" - }, - { - "name": "insuranceVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedInsuranceFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "feeVault", - "type": "publicKey" - }, - { - "name": "feeVaultBump", - "type": "u8" - }, - { - "name": "feeVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedGroupFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalLiabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalAssetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "lastUpdate", - "type": "i64" - }, - { - "name": "config", - "type": { - "defined": "BankConfig" - } - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - } - ], - "types": [ - { - "name": "GroupEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "version", - "type": "u16" - }, - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "AccountEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "version", - "type": "u16" - }, - { - "name": "signer", - "type": "publicKey" - }, - { - "name": "marginfiAccount", - "type": "publicKey" - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "LendingAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "balances", - "type": { - "array": [ - { - "defined": "Balance" - }, - 16 - ] - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 8 - ] - } - } - ] - } - }, - { - "name": "Balance", - "type": { - "kind": "struct", - "fields": [ - { - "name": "active", - "type": "bool" - }, - { - "name": "bankPk", - "type": "publicKey" - }, - { - "name": "assetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 4 - ] - } - } - ] - } - }, - { - "name": "GroupConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": { - "option": "publicKey" - } - } - ] - } - }, - { - "name": "InterestRateConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "plateauInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "maxInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u128", - 8 - ] - } - } - ] - } - }, - { - "name": "BankConfig", - "docs": [ - "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" - ], - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "assetWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "depositLimit", - "type": "u64" - }, - { - "name": "interestRateConfig", - "type": { - "defined": "InterestRateConfig" - } - }, - { - "name": "operationalState", - "type": { - "defined": "BankOperationalState" - } - }, - { - "name": "oracleSetup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "oracleKeys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - }, - { - "name": "borrowLimit", - "type": "u64" - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 7 - ] - } - } - ] - } - }, - { - "name": "WrappedI80F48", - "type": { - "kind": "struct", - "fields": [ - { - "name": "value", - "type": "i128" - } - ] - } - }, - { - "name": "BankConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "assetWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "depositLimit", - "type": { - "option": "u64" - } - }, - { - "name": "borrowLimit", - "type": { - "option": "u64" - } - }, - { - "name": "operationalState", - "type": { - "option": { - "defined": "BankOperationalState" - } - } - }, - { - "name": "oracle", - "type": { - "option": { - "defined": "OracleConfig" - } - } - } - ] - } - }, - { - "name": "OracleConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "setup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "keys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - } - ] - } - }, - { - "name": "BalanceIncreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "RepayOnly" - }, - { - "name": "DepositOnly" - } - ] - } - }, - { - "name": "BalanceDecreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "WithdrawOnly" - }, - { - "name": "BorrowOnly" - }, - { - "name": "BypassBorrowLimit" - } - ] - } - }, - { - "name": "WeightType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BalanceSide", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Assets" - }, - { - "name": "Liabilities" - } - ] - } - }, - { - "name": "RiskRequirementType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BankOperationalState", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Paused" - }, - { - "name": "Operational" - }, - { - "name": "ReduceOnly" - } - ] - } - }, - { - "name": "OracleSetup", - "type": { - "kind": "enum", - "variants": [ - { - "name": "None" - }, - { - "name": "Pyth" - } - ] - } - }, - { - "name": "OracleKey", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Pyth", - "fields": [ - "publicKey" - ] - } - ] - } - }, - { - "name": "BankVaultType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Liquidity" - }, - { - "name": "Insurance" - }, - { - "name": "Fee" - } - ] - } - } - ], - "events": [ - { - "name": "LendingPoolBankAddEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - } - ] - }, - { - "name": "LendingPoolBankAccrueInterestEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "delta", - "type": "u64", - "index": false - }, - { - "name": "feesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceCollected", - "type": "f64", - "index": false - } - ] - }, - { - "name": "MarginfiAccountCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - } - ] - }, - { - "name": "LendingAccountDepositEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountRepayEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountBorrowEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountWithdrawEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - } - ], - "errors": [ - { - "code": 6000, - "name": "MathError", - "msg": "Math error" - }, - { - "code": 6001, - "name": "BankNotFound", - "msg": "Invalid bank index" - }, - { - "code": 6002, - "name": "LendingAccountBalanceNotFound", - "msg": "Lending account balance not found" - }, - { - "code": 6003, - "name": "BankAssetCapacityExceeded", - "msg": "Bank deposit capacity exceeded" - }, - { - "code": 6004, - "name": "InvalidTransfer", - "msg": "Invalid transfer" - }, - { - "code": 6005, - "name": "MissingPythOrBankAccount", - "msg": "Missing Pyth or Bank account" - }, - { - "code": 6006, - "name": "MissingPythAccount", - "msg": "Missing Pyth account" - }, - { - "code": 6007, - "name": "InvalidOracleAccount", - "msg": "Invalid Pyth account" - }, - { - "code": 6008, - "name": "MissingBankAccount", - "msg": "Missing Bank account" - }, - { - "code": 6009, - "name": "InvalidBankAccount", - "msg": "Invalid Bank account" - }, - { - "code": 6010, - "name": "BadAccountHealth", - "msg": "Bad account health" - }, - { - "code": 6011, - "name": "LendingAccountBalanceSlotsFull", - "msg": "Lending account balance slots are full" - }, - { - "code": 6012, - "name": "BankAlreadyExists", - "msg": "Bank already exists" - }, - { - "code": 6013, - "name": "IllegalLiquidation", - "msg": "Illegal post liquidation state, account is either not unhealthy or liquidation was too big" - }, - { - "code": 6014, - "name": "AccountNotBankrupt", - "msg": "Account is not bankrupt" - }, - { - "code": 6015, - "name": "BalanceNotBadDebt", - "msg": "Account balance is not bad debt" - }, - { - "code": 6016, - "name": "InvalidConfig", - "msg": "Invalid group config" - }, - { - "code": 6017, - "name": "StaleOracle", - "msg": "Stale oracle data" - }, - { - "code": 6018, - "name": "BankPaused", - "msg": "Bank paused" - }, - { - "code": 6019, - "name": "BankReduceOnly", - "msg": "Bank is ReduceOnly mode" - }, - { - "code": 6020, - "name": "BankAccoutNotFound", - "msg": "Bank is missing" - }, - { - "code": 6021, - "name": "OperationDepositOnly", - "msg": "Operation is deposit-only" - }, - { - "code": 6022, - "name": "OperationWithdrawOnly", - "msg": "Operation is withdraw-only" - }, - { - "code": 6023, - "name": "OperationBorrowOnly", - "msg": "Operation is borrow-only" - }, - { - "code": 6024, - "name": "OperationRepayOnly", - "msg": "Operation is repay-only" - }, - { - "code": 6025, - "name": "NoAssetFound", - "msg": "No asset found" - }, - { - "code": 6026, - "name": "NoLiabilityFound", - "msg": "No liability found" - }, - { - "code": 6027, - "name": "InvalidOracleSetup", - "msg": "Invalid oracle setup" - }, - { - "code": 6028, - "name": "IllegalUtilizationRatio", - "msg": "Invalid bank utilization ratio" - }, - { - "code": 6029, - "name": "BankLiabilityCapacityExceeded", - "msg": "Bank borrow cap exceeded" - } - ], - "metadata": { - "address": "A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4" - } -} \ No newline at end of file diff --git a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v2.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v2.json deleted file mode 100644 index e1cfe965..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v2.json +++ /dev/null @@ -1,1899 +0,0 @@ -{ - "version": "0.1.0", - "name": "marginfi", - "instructions": [ - { - "name": "marginfiGroupInitialize", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": true - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiGroupConfigure", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - } - ], - "args": [ - { - "name": "config", - "type": { - "defined": "GroupConfig" - } - } - ] - }, - { - "name": "lendingPoolAddBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "bankMint", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": true - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "rent", - "isMut": false, - "isSigner": false - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfig", - "type": { - "defined": "BankConfig" - } - } - ] - }, - { - "name": "lendingPoolConfigureBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfigOpt", - "type": { - "defined": "BankConfigOpt" - } - } - ] - }, - { - "name": "lendingPoolHandleBankruptcy", - "docs": [ - "Handle bad debt of a bankrupt marginfi account for a given bank." - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiAccountInitialize", - "docs": [ - "Initialize a marginfi account for a given group" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": true - }, - { - "name": "authority", - "isMut": false, - "isSigner": true - }, - { - "name": "feePayer", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingAccountDeposit", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountRepay", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "repayAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountWithdraw", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "withdrawAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountBorrow", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountLiquidate", - "docs": [ - "Liquidate a lending account balance of an unhealthy marginfi account" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "assetBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liabBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidatorMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "liquidateeMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankInsuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "assetAmount", - "type": "u64" - } - ] - }, - { - "name": "lendingPoolAccrueBankInterest", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingPoolCollectBankFees", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - } - ], - "accounts": [ - { - "name": "MarginfiAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "group", - "type": "publicKey" - }, - { - "name": "authority", - "type": "publicKey" - }, - { - "name": "lendingAccount", - "type": { - "defined": "LendingAccount" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 64 - ] - } - } - ] - } - }, - { - "name": "MarginfiGroup", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": "publicKey" - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - }, - { - "name": "Bank", - "type": { - "kind": "struct", - "fields": [ - { - "name": "mint", - "type": "publicKey" - }, - { - "name": "mintDecimals", - "type": "u8" - }, - { - "name": "group", - "type": "publicKey" - }, - { - "name": "assetShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liquidityVault", - "type": "publicKey" - }, - { - "name": "liquidityVaultBump", - "type": "u8" - }, - { - "name": "liquidityVaultAuthorityBump", - "type": "u8" - }, - { - "name": "insuranceVault", - "type": "publicKey" - }, - { - "name": "insuranceVaultBump", - "type": "u8" - }, - { - "name": "insuranceVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedInsuranceFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "feeVault", - "type": "publicKey" - }, - { - "name": "feeVaultBump", - "type": "u8" - }, - { - "name": "feeVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedGroupFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalLiabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalAssetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "lastUpdate", - "type": "i64" - }, - { - "name": "config", - "type": { - "defined": "BankConfig" - } - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - } - ], - "types": [ - { - "name": "GroupEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "AccountEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": "publicKey" - }, - { - "name": "marginfiAccount", - "type": "publicKey" - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "LendingAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "balances", - "type": { - "array": [ - { - "defined": "Balance" - }, - 16 - ] - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 8 - ] - } - } - ] - } - }, - { - "name": "Balance", - "type": { - "kind": "struct", - "fields": [ - { - "name": "active", - "type": "bool" - }, - { - "name": "bankPk", - "type": "publicKey" - }, - { - "name": "assetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 4 - ] - } - } - ] - } - }, - { - "name": "GroupConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": { - "option": "publicKey" - } - } - ] - } - }, - { - "name": "InterestRateConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "plateauInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "maxInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u128", - 8 - ] - } - } - ] - } - }, - { - "name": "BankConfig", - "docs": [ - "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" - ], - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "assetWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "depositLimit", - "type": "u64" - }, - { - "name": "interestRateConfig", - "type": { - "defined": "InterestRateConfig" - } - }, - { - "name": "operationalState", - "type": { - "defined": "BankOperationalState" - } - }, - { - "name": "oracleSetup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "oracleKeys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - }, - { - "name": "borrowLimit", - "type": "u64" - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 7 - ] - } - } - ] - } - }, - { - "name": "WrappedI80F48", - "type": { - "kind": "struct", - "fields": [ - { - "name": "value", - "type": "i128" - } - ] - } - }, - { - "name": "BankConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "assetWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "depositLimit", - "type": { - "option": "u64" - } - }, - { - "name": "borrowLimit", - "type": { - "option": "u64" - } - }, - { - "name": "operationalState", - "type": { - "option": { - "defined": "BankOperationalState" - } - } - }, - { - "name": "oracle", - "type": { - "option": { - "defined": "OracleConfig" - } - } - } - ] - } - }, - { - "name": "OracleConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "setup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "keys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - } - ] - } - }, - { - "name": "BalanceIncreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "RepayOnly" - }, - { - "name": "DepositOnly" - } - ] - } - }, - { - "name": "BalanceDecreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "WithdrawOnly" - }, - { - "name": "BorrowOnly" - }, - { - "name": "BypassBorrowLimit" - } - ] - } - }, - { - "name": "WeightType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BalanceSide", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Assets" - }, - { - "name": "Liabilities" - } - ] - } - }, - { - "name": "RiskRequirementType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BankOperationalState", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Paused" - }, - { - "name": "Operational" - }, - { - "name": "ReduceOnly" - } - ] - } - }, - { - "name": "OracleSetup", - "type": { - "kind": "enum", - "variants": [ - { - "name": "None" - }, - { - "name": "Pyth" - } - ] - } - }, - { - "name": "OracleKey", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Pyth", - "fields": [ - "publicKey" - ] - } - ] - } - }, - { - "name": "BankVaultType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Liquidity" - }, - { - "name": "Insurance" - }, - { - "name": "Fee" - } - ] - } - } - ], - "events": [ - { - "name": "LendingPoolBankAddEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - } - ] - }, - { - "name": "LendingPoolBankAccrueInterestEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "delta", - "type": "u64", - "index": false - }, - { - "name": "feesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceCollected", - "type": "f64", - "index": false - } - ] - }, - { - "name": "MarginfiAccountCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - } - ] - }, - { - "name": "LendingAccountDepositEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountRepayEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountBorrowEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountWithdrawEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - } - ], - "errors": [ - { - "code": 6000, - "name": "MathError", - "msg": "Math error" - }, - { - "code": 6001, - "name": "BankNotFound", - "msg": "Invalid bank index" - }, - { - "code": 6002, - "name": "LendingAccountBalanceNotFound", - "msg": "Lending account balance not found" - }, - { - "code": 6003, - "name": "BankAssetCapacityExceeded", - "msg": "Bank deposit capacity exceeded" - }, - { - "code": 6004, - "name": "InvalidTransfer", - "msg": "Invalid transfer" - }, - { - "code": 6005, - "name": "MissingPythOrBankAccount", - "msg": "Missing Pyth or Bank account" - }, - { - "code": 6006, - "name": "MissingPythAccount", - "msg": "Missing Pyth account" - }, - { - "code": 6007, - "name": "InvalidOracleAccount", - "msg": "Invalid Pyth account" - }, - { - "code": 6008, - "name": "MissingBankAccount", - "msg": "Missing Bank account" - }, - { - "code": 6009, - "name": "InvalidBankAccount", - "msg": "Invalid Bank account" - }, - { - "code": 6010, - "name": "BadAccountHealth", - "msg": "Bad account health" - }, - { - "code": 6011, - "name": "LendingAccountBalanceSlotsFull", - "msg": "Lending account balance slots are full" - }, - { - "code": 6012, - "name": "BankAlreadyExists", - "msg": "Bank already exists" - }, - { - "code": 6013, - "name": "IllegalLiquidation", - "msg": "Illegal post liquidation state, account is either not unhealthy or liquidation was too big" - }, - { - "code": 6014, - "name": "AccountNotBankrupt", - "msg": "Account is not bankrupt" - }, - { - "code": 6015, - "name": "BalanceNotBadDebt", - "msg": "Account balance is not bad debt" - }, - { - "code": 6016, - "name": "InvalidConfig", - "msg": "Invalid group config" - }, - { - "code": 6017, - "name": "StaleOracle", - "msg": "Stale oracle data" - }, - { - "code": 6018, - "name": "BankPaused", - "msg": "Bank paused" - }, - { - "code": 6019, - "name": "BankReduceOnly", - "msg": "Bank is ReduceOnly mode" - }, - { - "code": 6020, - "name": "BankAccoutNotFound", - "msg": "Bank is missing" - }, - { - "code": 6021, - "name": "OperationDepositOnly", - "msg": "Operation is deposit-only" - }, - { - "code": 6022, - "name": "OperationWithdrawOnly", - "msg": "Operation is withdraw-only" - }, - { - "code": 6023, - "name": "OperationBorrowOnly", - "msg": "Operation is borrow-only" - }, - { - "code": 6024, - "name": "OperationRepayOnly", - "msg": "Operation is repay-only" - }, - { - "code": 6025, - "name": "NoAssetFound", - "msg": "No asset found" - }, - { - "code": 6026, - "name": "NoLiabilityFound", - "msg": "No liability found" - }, - { - "code": 6027, - "name": "InvalidOracleSetup", - "msg": "Invalid oracle setup" - }, - { - "code": 6028, - "name": "IllegalUtilizationRatio", - "msg": "Invalid bank utilization ratio" - }, - { - "code": 6029, - "name": "BankLiabilityCapacityExceeded", - "msg": "Bank borrow cap exceeded" - } - ], - "metadata": { - "address": "A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4" - } -} \ No newline at end of file diff --git a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v3.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v3.json deleted file mode 100644 index a075a088..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v3.json +++ /dev/null @@ -1,2032 +0,0 @@ -{ - "version": "0.1.0", - "name": "marginfi", - "instructions": [ - { - "name": "marginfiGroupInitialize", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": true - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiGroupConfigure", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - } - ], - "args": [ - { - "name": "config", - "type": { - "defined": "GroupConfig" - } - } - ] - }, - { - "name": "lendingPoolAddBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "bankMint", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": true - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "rent", - "isMut": false, - "isSigner": false - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfig", - "type": { - "defined": "BankConfig" - } - } - ] - }, - { - "name": "lendingPoolConfigureBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfigOpt", - "type": { - "defined": "BankConfigOpt" - } - } - ] - }, - { - "name": "lendingPoolHandleBankruptcy", - "docs": [ - "Handle bad debt of a bankrupt marginfi account for a given bank." - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiAccountInitialize", - "docs": [ - "Initialize a marginfi account for a given group" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": true - }, - { - "name": "authority", - "isMut": false, - "isSigner": true - }, - { - "name": "feePayer", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingAccountDeposit", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountRepay", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "repayAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountWithdraw", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "withdrawAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountBorrow", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountLiquidate", - "docs": [ - "Liquidate a lending account balance of an unhealthy marginfi account" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "assetBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liabBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidatorMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "liquidateeMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankInsuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "assetAmount", - "type": "u64" - } - ] - }, - { - "name": "lendingPoolAccrueBankInterest", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingPoolCollectBankFees", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - } - ], - "accounts": [ - { - "name": "MarginfiAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "group", - "type": "publicKey" - }, - { - "name": "authority", - "type": "publicKey" - }, - { - "name": "lendingAccount", - "type": { - "defined": "LendingAccount" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 64 - ] - } - } - ] - } - }, - { - "name": "MarginfiGroup", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": "publicKey" - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - }, - { - "name": "Bank", - "type": { - "kind": "struct", - "fields": [ - { - "name": "mint", - "type": "publicKey" - }, - { - "name": "mintDecimals", - "type": "u8" - }, - { - "name": "group", - "type": "publicKey" - }, - { - "name": "assetShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liquidityVault", - "type": "publicKey" - }, - { - "name": "liquidityVaultBump", - "type": "u8" - }, - { - "name": "liquidityVaultAuthorityBump", - "type": "u8" - }, - { - "name": "insuranceVault", - "type": "publicKey" - }, - { - "name": "insuranceVaultBump", - "type": "u8" - }, - { - "name": "insuranceVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedInsuranceFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "feeVault", - "type": "publicKey" - }, - { - "name": "feeVaultBump", - "type": "u8" - }, - { - "name": "feeVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedGroupFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalLiabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalAssetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "lastUpdate", - "type": "i64" - }, - { - "name": "config", - "type": { - "defined": "BankConfig" - } - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - } - ], - "types": [ - { - "name": "GroupEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "AccountEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": "publicKey" - }, - { - "name": "marginfiAccount", - "type": "publicKey" - }, - { - "name": "marginfiAccountAuthority", - "type": "publicKey" - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "LiquidationBalances", - "type": { - "kind": "struct", - "fields": [ - { - "name": "liquidateeAssetBalance", - "type": "f64" - }, - { - "name": "liquidateeLiabilityBalance", - "type": "f64" - }, - { - "name": "liquidatorAssetBalance", - "type": "f64" - }, - { - "name": "liquidatorLiabilityBalance", - "type": "f64" - } - ] - } - }, - { - "name": "LendingAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "balances", - "type": { - "array": [ - { - "defined": "Balance" - }, - 16 - ] - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 8 - ] - } - } - ] - } - }, - { - "name": "Balance", - "type": { - "kind": "struct", - "fields": [ - { - "name": "active", - "type": "bool" - }, - { - "name": "bankPk", - "type": "publicKey" - }, - { - "name": "assetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 4 - ] - } - } - ] - } - }, - { - "name": "GroupConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": { - "option": "publicKey" - } - } - ] - } - }, - { - "name": "InterestRateConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "plateauInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "maxInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u128", - 8 - ] - } - } - ] - } - }, - { - "name": "BankConfig", - "docs": [ - "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" - ], - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "assetWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "depositLimit", - "type": "u64" - }, - { - "name": "interestRateConfig", - "type": { - "defined": "InterestRateConfig" - } - }, - { - "name": "operationalState", - "type": { - "defined": "BankOperationalState" - } - }, - { - "name": "oracleSetup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "oracleKeys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - }, - { - "name": "borrowLimit", - "type": "u64" - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 7 - ] - } - } - ] - } - }, - { - "name": "WrappedI80F48", - "type": { - "kind": "struct", - "fields": [ - { - "name": "value", - "type": "i128" - } - ] - } - }, - { - "name": "BankConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "assetWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "depositLimit", - "type": { - "option": "u64" - } - }, - { - "name": "borrowLimit", - "type": { - "option": "u64" - } - }, - { - "name": "operationalState", - "type": { - "option": { - "defined": "BankOperationalState" - } - } - }, - { - "name": "oracle", - "type": { - "option": { - "defined": "OracleConfig" - } - } - } - ] - } - }, - { - "name": "OracleConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "setup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "keys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - } - ] - } - }, - { - "name": "BalanceIncreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "RepayOnly" - }, - { - "name": "DepositOnly" - } - ] - } - }, - { - "name": "BalanceDecreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "WithdrawOnly" - }, - { - "name": "BorrowOnly" - }, - { - "name": "BypassBorrowLimit" - } - ] - } - }, - { - "name": "WeightType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BalanceSide", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Assets" - }, - { - "name": "Liabilities" - } - ] - } - }, - { - "name": "RiskRequirementType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BankOperationalState", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Paused" - }, - { - "name": "Operational" - }, - { - "name": "ReduceOnly" - } - ] - } - }, - { - "name": "OracleSetup", - "type": { - "kind": "enum", - "variants": [ - { - "name": "None" - }, - { - "name": "Pyth" - } - ] - } - }, - { - "name": "OracleKey", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Pyth", - "fields": [ - "publicKey" - ] - } - ] - } - }, - { - "name": "BankVaultType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Liquidity" - }, - { - "name": "Insurance" - }, - { - "name": "Fee" - } - ] - } - } - ], - "events": [ - { - "name": "LendingPoolBankAddEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - } - ] - }, - { - "name": "LendingPoolBankAccrueInterestEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "delta", - "type": "u64", - "index": false - }, - { - "name": "feesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceCollected", - "type": "f64", - "index": false - } - ] - }, - { - "name": "MarginfiAccountCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - } - ] - }, - { - "name": "LendingAccountDepositEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountRepayEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountBorrowEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountWithdrawEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingPoolHandleBankruptcyEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "badDebt", - "type": "f64", - "index": false - }, - { - "name": "coveredAmount", - "type": "f64", - "index": false - }, - { - "name": "socializedAmount", - "type": "f64", - "index": false - } - ] - }, - { - "name": "LendingAccountLiquidateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "liquidateeMarginfiAccount", - "type": "publicKey", - "index": false - }, - { - "name": "liquidateeMarginfiAccountAuthority", - "type": "publicKey", - "index": false - }, - { - "name": "assetBank", - "type": "publicKey", - "index": false - }, - { - "name": "assetMint", - "type": "publicKey", - "index": false - }, - { - "name": "liabilityBank", - "type": "publicKey", - "index": false - }, - { - "name": "liabilityMint", - "type": "publicKey", - "index": false - }, - { - "name": "liquidateePreHealth", - "type": "f64", - "index": false - }, - { - "name": "liquidateePostHealth", - "type": "f64", - "index": false - }, - { - "name": "preBalances", - "type": { - "defined": "LiquidationBalances" - }, - "index": false - }, - { - "name": "postBalances", - "type": { - "defined": "LiquidationBalances" - }, - "index": false - } - ] - } - ], - "errors": [ - { - "code": 6000, - "name": "MathError", - "msg": "Math error" - }, - { - "code": 6001, - "name": "BankNotFound", - "msg": "Invalid bank index" - }, - { - "code": 6002, - "name": "LendingAccountBalanceNotFound", - "msg": "Lending account balance not found" - }, - { - "code": 6003, - "name": "BankAssetCapacityExceeded", - "msg": "Bank deposit capacity exceeded" - }, - { - "code": 6004, - "name": "InvalidTransfer", - "msg": "Invalid transfer" - }, - { - "code": 6005, - "name": "MissingPythOrBankAccount", - "msg": "Missing Pyth or Bank account" - }, - { - "code": 6006, - "name": "MissingPythAccount", - "msg": "Missing Pyth account" - }, - { - "code": 6007, - "name": "InvalidOracleAccount", - "msg": "Invalid Pyth account" - }, - { - "code": 6008, - "name": "MissingBankAccount", - "msg": "Missing Bank account" - }, - { - "code": 6009, - "name": "InvalidBankAccount", - "msg": "Invalid Bank account" - }, - { - "code": 6010, - "name": "BadAccountHealth", - "msg": "Bad account health" - }, - { - "code": 6011, - "name": "LendingAccountBalanceSlotsFull", - "msg": "Lending account balance slots are full" - }, - { - "code": 6012, - "name": "BankAlreadyExists", - "msg": "Bank already exists" - }, - { - "code": 6013, - "name": "IllegalLiquidation", - "msg": "Illegal post liquidation state, account is either not unhealthy or liquidation was too big" - }, - { - "code": 6014, - "name": "AccountNotBankrupt", - "msg": "Account is not bankrupt" - }, - { - "code": 6015, - "name": "BalanceNotBadDebt", - "msg": "Account balance is not bad debt" - }, - { - "code": 6016, - "name": "InvalidConfig", - "msg": "Invalid group config" - }, - { - "code": 6017, - "name": "StaleOracle", - "msg": "Stale oracle data" - }, - { - "code": 6018, - "name": "BankPaused", - "msg": "Bank paused" - }, - { - "code": 6019, - "name": "BankReduceOnly", - "msg": "Bank is ReduceOnly mode" - }, - { - "code": 6020, - "name": "BankAccoutNotFound", - "msg": "Bank is missing" - }, - { - "code": 6021, - "name": "OperationDepositOnly", - "msg": "Operation is deposit-only" - }, - { - "code": 6022, - "name": "OperationWithdrawOnly", - "msg": "Operation is withdraw-only" - }, - { - "code": 6023, - "name": "OperationBorrowOnly", - "msg": "Operation is borrow-only" - }, - { - "code": 6024, - "name": "OperationRepayOnly", - "msg": "Operation is repay-only" - }, - { - "code": 6025, - "name": "NoAssetFound", - "msg": "No asset found" - }, - { - "code": 6026, - "name": "NoLiabilityFound", - "msg": "No liability found" - }, - { - "code": 6027, - "name": "InvalidOracleSetup", - "msg": "Invalid oracle setup" - }, - { - "code": 6028, - "name": "IllegalUtilizationRatio", - "msg": "Invalid bank utilization ratio" - }, - { - "code": 6029, - "name": "BankLiabilityCapacityExceeded", - "msg": "Bank borrow cap exceeded" - } - ] -} \ No newline at end of file diff --git a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v4.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v4.json deleted file mode 100644 index db9029fb..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v4.json +++ /dev/null @@ -1,2216 +0,0 @@ -{ - "version": "0.1.0", - "name": "marginfi", - "instructions": [ - { - "name": "marginfiGroupInitialize", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": true - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiGroupConfigure", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - } - ], - "args": [ - { - "name": "config", - "type": { - "defined": "GroupConfig" - } - } - ] - }, - { - "name": "lendingPoolAddBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "bankMint", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": true - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "rent", - "isMut": false, - "isSigner": false - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfig", - "type": { - "defined": "BankConfig" - } - } - ] - }, - { - "name": "lendingPoolConfigureBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfigOpt", - "type": { - "defined": "BankConfigOpt" - } - } - ] - }, - { - "name": "lendingPoolHandleBankruptcy", - "docs": [ - "Handle bad debt of a bankrupt marginfi account for a given bank." - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiAccountInitialize", - "docs": [ - "Initialize a marginfi account for a given group" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": true - }, - { - "name": "authority", - "isMut": false, - "isSigner": true - }, - { - "name": "feePayer", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingAccountDeposit", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountRepay", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "repayAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountWithdraw", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "withdrawAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountBorrow", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountLiquidate", - "docs": [ - "Liquidate a lending account balance of an unhealthy marginfi account" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "assetBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liabBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidatorMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "liquidateeMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankInsuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "assetAmount", - "type": "u64" - } - ] - }, - { - "name": "lendingPoolAccrueBankInterest", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingPoolCollectBankFees", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - } - ], - "accounts": [ - { - "name": "MarginfiAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "group", - "type": "publicKey" - }, - { - "name": "authority", - "type": "publicKey" - }, - { - "name": "lendingAccount", - "type": { - "defined": "LendingAccount" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 64 - ] - } - } - ] - } - }, - { - "name": "MarginfiGroup", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": "publicKey" - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - }, - { - "name": "Bank", - "type": { - "kind": "struct", - "fields": [ - { - "name": "mint", - "type": "publicKey" - }, - { - "name": "mintDecimals", - "type": "u8" - }, - { - "name": "group", - "type": "publicKey" - }, - { - "name": "assetShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liquidityVault", - "type": "publicKey" - }, - { - "name": "liquidityVaultBump", - "type": "u8" - }, - { - "name": "liquidityVaultAuthorityBump", - "type": "u8" - }, - { - "name": "insuranceVault", - "type": "publicKey" - }, - { - "name": "insuranceVaultBump", - "type": "u8" - }, - { - "name": "insuranceVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedInsuranceFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "feeVault", - "type": "publicKey" - }, - { - "name": "feeVaultBump", - "type": "u8" - }, - { - "name": "feeVaultAuthorityBump", - "type": "u8" - }, - { - "name": "collectedGroupFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalLiabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalAssetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "lastUpdate", - "type": "i64" - }, - { - "name": "config", - "type": { - "defined": "BankConfig" - } - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - } - ], - "types": [ - { - "name": "GroupEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "AccountEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiAccount", - "type": "publicKey" - }, - { - "name": "marginfiAccountAuthority", - "type": "publicKey" - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "LiquidationBalances", - "type": { - "kind": "struct", - "fields": [ - { - "name": "liquidateeAssetBalance", - "type": "f64" - }, - { - "name": "liquidateeLiabilityBalance", - "type": "f64" - }, - { - "name": "liquidatorAssetBalance", - "type": "f64" - }, - { - "name": "liquidatorLiabilityBalance", - "type": "f64" - } - ] - } - }, - { - "name": "LendingAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "balances", - "type": { - "array": [ - { - "defined": "Balance" - }, - 16 - ] - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 8 - ] - } - } - ] - } - }, - { - "name": "Balance", - "type": { - "kind": "struct", - "fields": [ - { - "name": "active", - "type": "bool" - }, - { - "name": "bankPk", - "type": "publicKey" - }, - { - "name": "assetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 4 - ] - } - } - ] - } - }, - { - "name": "GroupConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": { - "option": "publicKey" - } - } - ] - } - }, - { - "name": "InterestRateConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "plateauInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "maxInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u128", - 8 - ] - } - } - ] - } - }, - { - "name": "InterestRateConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "plateauInterestRate", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "maxInterestRate", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "insuranceIrFee", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "protocolIrFee", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - } - ] - } - }, - { - "name": "BankConfig", - "docs": [ - "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" - ], - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "assetWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "depositLimit", - "type": "u64" - }, - { - "name": "interestRateConfig", - "type": { - "defined": "InterestRateConfig" - } - }, - { - "name": "operationalState", - "type": { - "defined": "BankOperationalState" - } - }, - { - "name": "oracleSetup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "oracleKeys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - }, - { - "name": "borrowLimit", - "type": "u64" - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 7 - ] - } - } - ] - } - }, - { - "name": "WrappedI80F48", - "type": { - "kind": "struct", - "fields": [ - { - "name": "value", - "type": "i128" - } - ] - } - }, - { - "name": "BankConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "assetWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "depositLimit", - "type": { - "option": "u64" - } - }, - { - "name": "borrowLimit", - "type": { - "option": "u64" - } - }, - { - "name": "operationalState", - "type": { - "option": { - "defined": "BankOperationalState" - } - } - }, - { - "name": "oracle", - "type": { - "option": { - "defined": "OracleConfig" - } - } - }, - { - "name": "interestRateConfig", - "type": { - "option": { - "defined": "InterestRateConfigOpt" - } - } - } - ] - } - }, - { - "name": "OracleConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "setup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "keys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - } - ] - } - }, - { - "name": "BalanceIncreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "RepayOnly" - }, - { - "name": "DepositOnly" - } - ] - } - }, - { - "name": "BalanceDecreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "WithdrawOnly" - }, - { - "name": "BorrowOnly" - }, - { - "name": "BypassBorrowLimit" - } - ] - } - }, - { - "name": "WeightType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BalanceSide", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Assets" - }, - { - "name": "Liabilities" - } - ] - } - }, - { - "name": "RiskRequirementType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BankOperationalState", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Paused" - }, - { - "name": "Operational" - }, - { - "name": "ReduceOnly" - } - ] - } - }, - { - "name": "OracleSetup", - "type": { - "kind": "enum", - "variants": [ - { - "name": "None" - }, - { - "name": "Pyth" - } - ] - } - }, - { - "name": "OracleKey", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Pyth", - "fields": [ - "publicKey" - ] - } - ] - } - }, - { - "name": "BankVaultType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Liquidity" - }, - { - "name": "Insurance" - }, - { - "name": "Fee" - } - ] - } - } - ], - "events": [ - { - "name": "MarginfiGroupCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - } - ] - }, - { - "name": "MarginfiGroupConfigureEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "config", - "type": { - "defined": "GroupConfig" - }, - "index": false - } - ] - }, - { - "name": "LendingPoolBankCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - } - ] - }, - { - "name": "LendingPoolBankConfigureEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "config", - "type": { - "defined": "BankConfigOpt" - }, - "index": false - } - ] - }, - { - "name": "LendingPoolBankAccrueInterestEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "delta", - "type": "u64", - "index": false - }, - { - "name": "feesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceCollected", - "type": "f64", - "index": false - } - ] - }, - { - "name": "LendingPoolBankCollectFeesEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "groupFeesCollected", - "type": "f64", - "index": false - }, - { - "name": "groupFeesOutstanding", - "type": "f64", - "index": false - }, - { - "name": "insuranceFeesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceFeesOutstanding", - "type": "f64", - "index": false - } - ] - }, - { - "name": "LendingPoolBankHandleBankruptcyEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "badDebt", - "type": "f64", - "index": false - }, - { - "name": "coveredAmount", - "type": "f64", - "index": false - }, - { - "name": "socializedAmount", - "type": "f64", - "index": false - } - ] - }, - { - "name": "MarginfiAccountCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - } - ] - }, - { - "name": "LendingAccountDepositEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountRepayEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountBorrowEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountWithdrawEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountLiquidateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "liquidateeMarginfiAccount", - "type": "publicKey", - "index": false - }, - { - "name": "liquidateeMarginfiAccountAuthority", - "type": "publicKey", - "index": false - }, - { - "name": "assetBank", - "type": "publicKey", - "index": false - }, - { - "name": "assetMint", - "type": "publicKey", - "index": false - }, - { - "name": "liabilityBank", - "type": "publicKey", - "index": false - }, - { - "name": "liabilityMint", - "type": "publicKey", - "index": false - }, - { - "name": "liquidateePreHealth", - "type": "f64", - "index": false - }, - { - "name": "liquidateePostHealth", - "type": "f64", - "index": false - }, - { - "name": "preBalances", - "type": { - "defined": "LiquidationBalances" - }, - "index": false - }, - { - "name": "postBalances", - "type": { - "defined": "LiquidationBalances" - }, - "index": false - } - ] - } - ], - "errors": [ - { - "code": 6000, - "name": "MathError", - "msg": "Math error" - }, - { - "code": 6001, - "name": "BankNotFound", - "msg": "Invalid bank index" - }, - { - "code": 6002, - "name": "LendingAccountBalanceNotFound", - "msg": "Lending account balance not found" - }, - { - "code": 6003, - "name": "BankAssetCapacityExceeded", - "msg": "Bank deposit capacity exceeded" - }, - { - "code": 6004, - "name": "InvalidTransfer", - "msg": "Invalid transfer" - }, - { - "code": 6005, - "name": "MissingPythOrBankAccount", - "msg": "Missing Pyth or Bank account" - }, - { - "code": 6006, - "name": "MissingPythAccount", - "msg": "Missing Pyth account" - }, - { - "code": 6007, - "name": "InvalidOracleAccount", - "msg": "Invalid Pyth account" - }, - { - "code": 6008, - "name": "MissingBankAccount", - "msg": "Missing Bank account" - }, - { - "code": 6009, - "name": "InvalidBankAccount", - "msg": "Invalid Bank account" - }, - { - "code": 6010, - "name": "BadAccountHealth", - "msg": "Bad account health" - }, - { - "code": 6011, - "name": "LendingAccountBalanceSlotsFull", - "msg": "Lending account balance slots are full" - }, - { - "code": 6012, - "name": "BankAlreadyExists", - "msg": "Bank already exists" - }, - { - "code": 6013, - "name": "IllegalLiquidation", - "msg": "Illegal post liquidation state, account is either not unhealthy or liquidation was too big" - }, - { - "code": 6014, - "name": "AccountNotBankrupt", - "msg": "Account is not bankrupt" - }, - { - "code": 6015, - "name": "BalanceNotBadDebt", - "msg": "Account balance is not bad debt" - }, - { - "code": 6016, - "name": "InvalidConfig", - "msg": "Invalid group config" - }, - { - "code": 6017, - "name": "StaleOracle", - "msg": "Stale oracle data" - }, - { - "code": 6018, - "name": "BankPaused", - "msg": "Bank paused" - }, - { - "code": 6019, - "name": "BankReduceOnly", - "msg": "Bank is ReduceOnly mode" - }, - { - "code": 6020, - "name": "BankAccoutNotFound", - "msg": "Bank is missing" - }, - { - "code": 6021, - "name": "OperationDepositOnly", - "msg": "Operation is deposit-only" - }, - { - "code": 6022, - "name": "OperationWithdrawOnly", - "msg": "Operation is withdraw-only" - }, - { - "code": 6023, - "name": "OperationBorrowOnly", - "msg": "Operation is borrow-only" - }, - { - "code": 6024, - "name": "OperationRepayOnly", - "msg": "Operation is repay-only" - }, - { - "code": 6025, - "name": "NoAssetFound", - "msg": "No asset found" - }, - { - "code": 6026, - "name": "NoLiabilityFound", - "msg": "No liability found" - }, - { - "code": 6027, - "name": "InvalidOracleSetup", - "msg": "Invalid oracle setup" - }, - { - "code": 6028, - "name": "IllegalUtilizationRatio", - "msg": "Invalid bank utilization ratio" - }, - { - "code": 6029, - "name": "BankLiabilityCapacityExceeded", - "msg": "Bank borrow cap exceeded" - }, - { - "code": 6030, - "name": "InvalidPrice", - "msg": "Invalid Price" - } - ], - "metadata": { - "address": "A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4" - } -} \ No newline at end of file diff --git a/observability/etl/dataflow-etls/dataflow_etls/idls/mainnet/MFv2hWf31Z9kbCa1snEPYctwafyhdvnV7FZnsebVacA/marginfi-v0.json b/observability/etl/dataflow-etls/dataflow_etls/idls/mainnet/MFv2hWf31Z9kbCa1snEPYctwafyhdvnV7FZnsebVacA/marginfi-v0.json deleted file mode 100644 index 50d97aa6..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/idls/mainnet/MFv2hWf31Z9kbCa1snEPYctwafyhdvnV7FZnsebVacA/marginfi-v0.json +++ /dev/null @@ -1,2478 +0,0 @@ -{ - "version": "0.1.0", - "name": "marginfi", - "instructions": [ - { - "name": "marginfiGroupInitialize", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": true - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiGroupConfigure", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": true, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - } - ], - "args": [ - { - "name": "config", - "type": { - "defined": "GroupConfig" - } - } - ] - }, - { - "name": "lendingPoolAddBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": true, - "isSigner": true - }, - { - "name": "bankMint", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": true - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "rent", - "isMut": false, - "isSigner": false - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfig", - "type": { - "defined": "BankConfigArg" - } - } - ] - }, - { - "name": "lendingPoolConfigureBank", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [ - { - "name": "bankConfigOpt", - "type": { - "defined": "BankConfigOptArg" - } - } - ] - }, - { - "name": "lendingPoolHandleBankruptcy", - "docs": [ - "Handle bad debt of a bankrupt marginfi account for a given bank." - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "admin", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "marginfiAccountInitialize", - "docs": [ - "Initialize a marginfi account for a given group" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": true - }, - { - "name": "authority", - "isMut": false, - "isSigner": true - }, - { - "name": "feePayer", - "isMut": true, - "isSigner": true - }, - { - "name": "systemProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingAccountDeposit", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountRepay", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "signerTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "repayAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountWithdraw", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - }, - { - "name": "withdrawAll", - "type": { - "option": "bool" - } - } - ] - }, - { - "name": "lendingAccountBorrow", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "marginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "destinationTokenAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "amount", - "type": "u64" - } - ] - }, - { - "name": "lendingAccountLiquidate", - "docs": [ - "Liquidate a lending account balance of an unhealthy marginfi account" - ], - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "assetBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liabBank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidatorMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "signer", - "isMut": false, - "isSigner": true - }, - { - "name": "liquidateeMarginfiAccount", - "isMut": true, - "isSigner": false - }, - { - "name": "bankLiquidityVaultAuthority", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankLiquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "bankInsuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "liab_bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [ - { - "name": "assetAmount", - "type": "u64" - } - ] - }, - { - "name": "lendingPoolAccrueBankInterest", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - } - ], - "args": [] - }, - { - "name": "lendingPoolCollectBankFees", - "accounts": [ - { - "name": "marginfiGroup", - "isMut": false, - "isSigner": false - }, - { - "name": "bank", - "isMut": true, - "isSigner": false - }, - { - "name": "liquidityVaultAuthority", - "isMut": false, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault_auth" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "liquidityVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "liquidity_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "insuranceVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "insurance_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "feeVault", - "isMut": true, - "isSigner": false, - "pda": { - "seeds": [ - { - "kind": "const", - "type": "string", - "value": "fee_vault" - }, - { - "kind": "account", - "type": "publicKey", - "path": "bank" - } - ] - } - }, - { - "name": "tokenProgram", - "isMut": false, - "isSigner": false - } - ], - "args": [] - } - ], - "accounts": [ - { - "name": "MarginfiAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "group", - "type": "publicKey" - }, - { - "name": "authority", - "type": "publicKey" - }, - { - "name": "lendingAccount", - "type": { - "defined": "LendingAccount" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 64 - ] - } - } - ] - } - }, - { - "name": "MarginfiGroup", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": "publicKey" - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - }, - { - "name": "Bank", - "type": { - "kind": "struct", - "fields": [ - { - "name": "mint", - "type": "publicKey" - }, - { - "name": "mintDecimals", - "type": "u8" - }, - { - "name": "group", - "type": "publicKey" - }, - { - "name": "ignore1", - "type": { - "array": [ - "u8", - 7 - ] - } - }, - { - "name": "assetShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShareValue", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liquidityVault", - "type": "publicKey" - }, - { - "name": "liquidityVaultBump", - "type": "u8" - }, - { - "name": "liquidityVaultAuthorityBump", - "type": "u8" - }, - { - "name": "insuranceVault", - "type": "publicKey" - }, - { - "name": "insuranceVaultBump", - "type": "u8" - }, - { - "name": "insuranceVaultAuthorityBump", - "type": "u8" - }, - { - "name": "ignore2", - "type": { - "array": [ - "u8", - 4 - ] - } - }, - { - "name": "collectedInsuranceFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "feeVault", - "type": "publicKey" - }, - { - "name": "feeVaultBump", - "type": "u8" - }, - { - "name": "feeVaultAuthorityBump", - "type": "u8" - }, - { - "name": "ignore3", - "type": { - "array": [ - "u8", - 6 - ] - } - }, - { - "name": "collectedGroupFeesOutstanding", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalLiabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "totalAssetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "lastUpdate", - "type": "i64" - }, - { - "name": "config", - "type": { - "defined": "BankConfig" - } - }, - { - "name": "padding0", - "type": { - "array": [ - "u128", - 32 - ] - } - }, - { - "name": "padding1", - "type": { - "array": [ - "u128", - 32 - ] - } - } - ] - } - } - ], - "types": [ - { - "name": "GroupEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "AccountEventHeader", - "type": { - "kind": "struct", - "fields": [ - { - "name": "signer", - "type": { - "option": "publicKey" - } - }, - { - "name": "marginfiAccount", - "type": "publicKey" - }, - { - "name": "marginfiAccountAuthority", - "type": "publicKey" - }, - { - "name": "marginfiGroup", - "type": "publicKey" - } - ] - } - }, - { - "name": "LiquidationBalances", - "type": { - "kind": "struct", - "fields": [ - { - "name": "liquidateeAssetBalance", - "type": "f64" - }, - { - "name": "liquidateeLiabilityBalance", - "type": "f64" - }, - { - "name": "liquidatorAssetBalance", - "type": "f64" - }, - { - "name": "liquidatorLiabilityBalance", - "type": "f64" - } - ] - } - }, - { - "name": "LendingAccount", - "type": { - "kind": "struct", - "fields": [ - { - "name": "balances", - "type": { - "array": [ - { - "defined": "Balance" - }, - 16 - ] - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 8 - ] - } - } - ] - } - }, - { - "name": "Balance", - "type": { - "kind": "struct", - "fields": [ - { - "name": "active", - "type": "bool" - }, - { - "name": "bankPk", - "type": "publicKey" - }, - { - "name": "ignore1", - "type": { - "array": [ - "u8", - 7 - ] - } - }, - { - "name": "assetShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityShares", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 4 - ] - } - } - ] - } - }, - { - "name": "GroupConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "admin", - "type": { - "option": "publicKey" - } - } - ] - } - }, - { - "name": "InterestRateConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "plateauInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "maxInterestRate", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "insuranceIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "protocolIrFee", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u128", - 8 - ] - } - } - ] - } - }, - { - "name": "InterestRateConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "optimalUtilizationRate", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "plateauInterestRate", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "maxInterestRate", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "insuranceFeeFixedApr", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "insuranceIrFee", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "protocolFixedFeeApr", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "protocolIrFee", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - } - ] - } - }, - { - "name": "BankConfig", - "docs": [ - "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" - ], - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "assetWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "depositLimit", - "type": "u64" - }, - { - "name": "interestRateConfig", - "type": { - "defined": "InterestRateConfig" - } - }, - { - "name": "operationalState", - "type": { - "defined": "BankOperationalState" - } - }, - { - "name": "oracleSetup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "oracleKeys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - }, - { - "name": "ignore1", - "type": { - "array": [ - "u64", - 6 - ] - } - }, - { - "name": "borrowLimit", - "type": "u64" - }, - { - "name": "riskTier", - "type": { - "defined": "RiskTier" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 6 - ] - } - } - ] - } - }, - { - "name": "BankConfigArg", - "docs": [ - "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" - ], - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "assetWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightInit", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "defined": "WrappedI80F48" - } - }, - { - "name": "depositLimit", - "type": "u64" - }, - { - "name": "interestRateConfig", - "type": { - "defined": "InterestRateConfig" - } - }, - { - "name": "operationalState", - "type": { - "defined": "BankOperationalState" - } - }, - { - "name": "oracleSetup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "oracleKeys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - }, - { - "name": "borrowLimit", - "type": "u64" - }, - { - "name": "riskTier", - "type": { - "defined": "RiskTier" - } - }, - { - "name": "padding", - "type": { - "array": [ - "u64", - 6 - ] - } - } - ] - } - }, - { - "name": "WrappedI80F48", - "type": { - "kind": "struct", - "fields": [ - { - "name": "value", - "type": "i128" - } - ] - } - }, - { - "name": "BankConfigOpt", - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "assetWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "depositLimit", - "type": { - "option": "u64" - } - }, - { - "name": "borrowLimit", - "type": { - "option": "u64" - } - }, - { - "name": "ignore1", - "type": { - "array": [ - "u64", - 176 - ] - } - }, - { - "name": "operationalState", - "type": { - "option": { - "defined": "BankOperationalState" - } - } - }, - { - "name": "oracle", - "type": { - "option": { - "defined": "OracleConfig" - } - } - }, - { - "name": "interestRateConfig", - "type": { - "option": { - "defined": "InterestRateConfigOpt" - } - } - }, - { - "name": "riskTier", - "type": { - "option": { - "defined": "RiskTier" - } - } - }, - { - "name": "ignore2", - "type": { - "array": [ - "u64", - 168 - ] - } - } - ] - } - }, - { - "name": "BankConfigOptArg", - "type": { - "kind": "struct", - "fields": [ - { - "name": "assetWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "assetWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightInit", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "liabilityWeightMaint", - "type": { - "option": { - "defined": "WrappedI80F48" - } - } - }, - { - "name": "depositLimit", - "type": { - "option": "u64" - } - }, - { - "name": "borrowLimit", - "type": { - "option": "u64" - } - }, - { - "name": "operationalState", - "type": { - "option": { - "defined": "BankOperationalState" - } - } - }, - { - "name": "oracle", - "type": { - "option": { - "defined": "OracleConfig" - } - } - }, - { - "name": "interestRateConfig", - "type": { - "option": { - "defined": "InterestRateConfigOpt" - } - } - }, - { - "name": "riskTier", - "type": { - "option": { - "defined": "RiskTier" - } - } - } - ] - } - }, - { - "name": "OracleConfig", - "type": { - "kind": "struct", - "fields": [ - { - "name": "setup", - "type": { - "defined": "OracleSetup" - } - }, - { - "name": "keys", - "type": { - "array": [ - "publicKey", - 5 - ] - } - } - ] - } - }, - { - "name": "BalanceIncreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "RepayOnly" - }, - { - "name": "DepositOnly" - } - ] - } - }, - { - "name": "BalanceDecreaseType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Any" - }, - { - "name": "WithdrawOnly" - }, - { - "name": "BorrowOnly" - }, - { - "name": "BypassBorrowLimit" - } - ] - } - }, - { - "name": "WeightType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BalanceSide", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Assets" - }, - { - "name": "Liabilities" - } - ] - } - }, - { - "name": "RiskRequirementType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Initial" - }, - { - "name": "Maintenance" - } - ] - } - }, - { - "name": "BankOperationalState", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Paused" - }, - { - "name": "Operational" - }, - { - "name": "ReduceOnly" - } - ] - } - }, - { - "name": "OracleSetup", - "type": { - "kind": "enum", - "variants": [ - { - "name": "None" - }, - { - "name": "Pyth" - } - ] - } - }, - { - "name": "OracleKey", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Pyth", - "fields": [ - "publicKey" - ] - } - ] - } - }, - { - "name": "RiskTier", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Collateral" - }, - { - "name": "Isolated" - } - ] - } - }, - { - "name": "BankVaultType", - "type": { - "kind": "enum", - "variants": [ - { - "name": "Liquidity" - }, - { - "name": "Insurance" - }, - { - "name": "Fee" - } - ] - } - } - ], - "events": [ - { - "name": "MarginfiGroupCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - } - ] - }, - { - "name": "MarginfiGroupConfigureEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "config", - "type": { - "defined": "GroupConfig" - }, - "index": false - } - ] - }, - { - "name": "LendingPoolBankCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - } - ] - }, - { - "name": "LendingPoolBankConfigureEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "config", - "type": { - "defined": "BankConfigOptArg" - }, - "index": false - } - ] - }, - { - "name": "LendingPoolBankAccrueInterestEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "delta", - "type": "u64", - "index": false - }, - { - "name": "feesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceCollected", - "type": "f64", - "index": false - } - ] - }, - { - "name": "LendingPoolBankCollectFeesEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "GroupEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "groupFeesCollected", - "type": "f64", - "index": false - }, - { - "name": "groupFeesOutstanding", - "type": "f64", - "index": false - }, - { - "name": "insuranceFeesCollected", - "type": "f64", - "index": false - }, - { - "name": "insuranceFeesOutstanding", - "type": "f64", - "index": false - } - ] - }, - { - "name": "LendingPoolBankHandleBankruptcyEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "badDebt", - "type": "f64", - "index": false - }, - { - "name": "coveredAmount", - "type": "f64", - "index": false - }, - { - "name": "socializedAmount", - "type": "f64", - "index": false - } - ] - }, - { - "name": "MarginfiAccountCreateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - } - ] - }, - { - "name": "LendingAccountDepositEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountRepayEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountBorrowEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - } - ] - }, - { - "name": "LendingAccountWithdrawEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "bank", - "type": "publicKey", - "index": false - }, - { - "name": "mint", - "type": "publicKey", - "index": false - }, - { - "name": "amount", - "type": "u64", - "index": false - }, - { - "name": "closeBalance", - "type": "bool", - "index": false - } - ] - }, - { - "name": "LendingAccountLiquidateEvent", - "fields": [ - { - "name": "header", - "type": { - "defined": "AccountEventHeader" - }, - "index": false - }, - { - "name": "liquidateeMarginfiAccount", - "type": "publicKey", - "index": false - }, - { - "name": "liquidateeMarginfiAccountAuthority", - "type": "publicKey", - "index": false - }, - { - "name": "assetBank", - "type": "publicKey", - "index": false - }, - { - "name": "assetMint", - "type": "publicKey", - "index": false - }, - { - "name": "liabilityBank", - "type": "publicKey", - "index": false - }, - { - "name": "liabilityMint", - "type": "publicKey", - "index": false - }, - { - "name": "liquidateePreHealth", - "type": "f64", - "index": false - }, - { - "name": "liquidateePostHealth", - "type": "f64", - "index": false - }, - { - "name": "preBalances", - "type": { - "defined": "LiquidationBalances" - }, - "index": false - }, - { - "name": "postBalances", - "type": { - "defined": "LiquidationBalances" - }, - "index": false - } - ] - } - ], - "errors": [ - { - "code": 6000, - "name": "MathError", - "msg": "Math error" - }, - { - "code": 6001, - "name": "BankNotFound", - "msg": "Invalid bank index" - }, - { - "code": 6002, - "name": "LendingAccountBalanceNotFound", - "msg": "Lending account balance not found" - }, - { - "code": 6003, - "name": "BankAssetCapacityExceeded", - "msg": "Bank deposit capacity exceeded" - }, - { - "code": 6004, - "name": "InvalidTransfer", - "msg": "Invalid transfer" - }, - { - "code": 6005, - "name": "MissingPythOrBankAccount", - "msg": "Missing Pyth or Bank account" - }, - { - "code": 6006, - "name": "MissingPythAccount", - "msg": "Missing Pyth account" - }, - { - "code": 6007, - "name": "InvalidOracleAccount", - "msg": "Invalid Pyth account" - }, - { - "code": 6008, - "name": "MissingBankAccount", - "msg": "Missing Bank account" - }, - { - "code": 6009, - "name": "InvalidBankAccount", - "msg": "Invalid Bank account" - }, - { - "code": 6010, - "name": "BadAccountHealth", - "msg": "Bad account health" - }, - { - "code": 6011, - "name": "LendingAccountBalanceSlotsFull", - "msg": "Lending account balance slots are full" - }, - { - "code": 6012, - "name": "BankAlreadyExists", - "msg": "Bank already exists" - }, - { - "code": 6013, - "name": "IllegalLiquidation", - "msg": "Illegal post liquidation state, account is either not unhealthy or liquidation was too big" - }, - { - "code": 6014, - "name": "AccountNotBankrupt", - "msg": "Account is not bankrupt" - }, - { - "code": 6015, - "name": "BalanceNotBadDebt", - "msg": "Account balance is not bad debt" - }, - { - "code": 6016, - "name": "InvalidConfig", - "msg": "Invalid group config" - }, - { - "code": 6017, - "name": "StaleOracle", - "msg": "Stale oracle data" - }, - { - "code": 6018, - "name": "BankPaused", - "msg": "Bank paused" - }, - { - "code": 6019, - "name": "BankReduceOnly", - "msg": "Bank is ReduceOnly mode" - }, - { - "code": 6020, - "name": "BankAccoutNotFound", - "msg": "Bank is missing" - }, - { - "code": 6021, - "name": "OperationDepositOnly", - "msg": "Operation is deposit-only" - }, - { - "code": 6022, - "name": "OperationWithdrawOnly", - "msg": "Operation is withdraw-only" - }, - { - "code": 6023, - "name": "OperationBorrowOnly", - "msg": "Operation is borrow-only" - }, - { - "code": 6024, - "name": "OperationRepayOnly", - "msg": "Operation is repay-only" - }, - { - "code": 6025, - "name": "NoAssetFound", - "msg": "No asset found" - }, - { - "code": 6026, - "name": "NoLiabilityFound", - "msg": "No liability found" - }, - { - "code": 6027, - "name": "InvalidOracleSetup", - "msg": "Invalid oracle setup" - }, - { - "code": 6028, - "name": "IllegalUtilizationRatio", - "msg": "Invalid bank utilization ratio" - }, - { - "code": 6029, - "name": "BankLiabilityCapacityExceeded", - "msg": "Bank borrow cap exceeded" - }, - { - "code": 6030, - "name": "InvalidPrice", - "msg": "Invalid Price" - }, - { - "code": 6031, - "name": "IsolatedAccountIllegalState", - "msg": "Account can have only one liablity when account is under isolated risk" - } - ] -} diff --git a/observability/etl/dataflow-etls/dataflow_etls/orm/__init__.py b/observability/etl/dataflow-etls/dataflow_etls/orm/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/observability/etl/dataflow-etls/dataflow_etls/orm/accounts.py b/observability/etl/dataflow-etls/dataflow_etls/orm/accounts.py deleted file mode 100644 index a4ab8a47..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/orm/accounts.py +++ /dev/null @@ -1,245 +0,0 @@ -import json -import uuid -from dataclasses import dataclass -from typing import Union, Dict, Type, TYPE_CHECKING -from anchorpy.program.common import NamedInstruction as NamedAccountData - -from dataflow_etls.utils import pascal_to_snake_case, time_str, wrapped_i80f48_to_float, enum_to_str - -if TYPE_CHECKING: - from dataflow_etls.account_parsing import AccountUpdateRaw - -# IDL account names -MARGINFI_GROUP_ACCOUNT_NAME = 'MarginfiGroup' -MARGINFI_ACCOUNT_ACCOUNT_NAME = 'MarginfiAccount' -LENDING_POOL_BANK_ACCOUNT_NAME = 'Bank' - - -@dataclass -class AccountUpdateRecordBase: - SCHEMA = ",".join( - [ - "id:STRING", - "created_at:TIMESTAMP", - "idl_version:INTEGER", - "timestamp:TIMESTAMP", - "owner_program:STRING", - "pubkey:STRING", - ] - ) - - id: str - created_at: str - idl_version: int - timestamp: str - owner_program: str - pubkey: str - - def __init__(self, _parsed_data: NamedAccountData, account_update: "AccountUpdateRaw", idl_version: int): - self.id = str(uuid.uuid4()) - self.created_at = time_str() - self.timestamp = time_str(account_update['timestamp']) - self.idl_version = idl_version - self.owner_program = str(account_update['owner']) - self.pubkey = str(account_update['pubkey']) - - @classmethod - def get_tag(cls, snake_case: bool = False) -> str: - if snake_case: - return pascal_to_snake_case(cls.__name__) - else: - return cls.__name__ - - -# Event headers - - -@dataclass -class MarginfiGroupUpdateRecord(AccountUpdateRecordBase): - SCHEMA = AccountUpdateRecordBase.SCHEMA + "," + ",".join( - [ - "admin:STRING", - ] - ) - - admin: str - - def __init__(self, parsed_data: NamedAccountData, account_update: "AccountUpdateRaw", idl_version: int): - super().__init__(parsed_data, account_update, idl_version) - - self.admin = str(parsed_data.data.admin) - - -@dataclass -class MarginfiAccountUpdateRecord(AccountUpdateRecordBase): - SCHEMA = AccountUpdateRecordBase.SCHEMA + "," + ",".join( - [ - "group:STRING", - "authority:STRING", - "active_balances:STRING", - ] - ) - - group: str - authority: str - active_balances: str - - def __init__(self, parsed_data: NamedAccountData, account_update: "AccountUpdateRaw", idl_version: int): - super().__init__(parsed_data, account_update, idl_version) - - self.group = str(parsed_data.data.group) - self.authority = str(parsed_data.data.authority) - self.active_balances = json.dumps([{"bank": str(balance.bank_pk), - "asset_shares": wrapped_i80f48_to_float(balance.asset_shares), - "liability_shares": wrapped_i80f48_to_float(balance.liability_shares) - } for balance in parsed_data.data.lending_account.balances if - balance.active]) - - -@dataclass -class LendingPoolBankUpdateRecord(AccountUpdateRecordBase): - SCHEMA = AccountUpdateRecordBase.SCHEMA + "," + ",".join( - [ - "mint:STRING", - "mint_decimals:INTEGER", - "group:STRING", - "asset_share_value:BIGNUMERIC", - "liability_share_value:BIGNUMERIC", - "liquidity_vault:STRING", - "liquidity_vault_bump:INTEGER", - "liquidity_vault_authority_bump:INTEGER", - "insurance_vault:STRING", - "insurance_vault_bump:INTEGER", - "insurance_vault_authority_bump:INTEGER", - "fee_vault:STRING", - "fee_vault_bump:INTEGER", - "fee_vault_authority_bump:INTEGER", - "collected_insurance_fees_outstanding:BIGNUMERIC", - "collected_group_fees_outstanding:BIGNUMERIC", - "total_liability_shares:BIGNUMERIC", - "total_asset_shares:BIGNUMERIC", - "last_update:BIGNUMERIC", - "config_asset_weight_init:BIGNUMERIC", - "config_asset_weight_maint:BIGNUMERIC", - "config_liability_weight_init:BIGNUMERIC", - "config_liability_weight_maint:BIGNUMERIC", - "config_deposit_limit:BIGNUMERIC", - "config_borrow_limit:BIGNUMERIC", - "config_interest_rate_config_optimal_utilization_rate:BIGNUMERIC", - "config_interest_rate_config_plateau_interest_rate:BIGNUMERIC", - "config_interest_rate_config_max_interest_rate:BIGNUMERIC", - "config_interest_rate_config_insurance_fee_fixed_apr:BIGNUMERIC", - "config_interest_rate_config_insurance_ir_fee:BIGNUMERIC", - "config_interest_rate_config_protocol_fixed_fee_apr:BIGNUMERIC", - "config_interest_rate_config_protocol_ir_fee:BIGNUMERIC", - "config_operational_state:STRING", - "config_oracle_setup:STRING", - "config_oracle_keys:STRING", - "config_risk_tier:STRING", - ] - ) - - mint: str - mint_decimals: int - group: str - asset_share_value: float - liability_share_value: float - liquidity_vault: str - liquidity_vault_bump: int - liquidity_vault_authority_bump: int - insurance_vault: str - insurance_vault_bump: int - insurance_vault_authority_bump: int - fee_vault: str - fee_vault_bump: int - fee_vault_authority_bump: int - collected_insurance_fees_outstanding: float - collected_group_fees_outstanding: float - total_liability_shares: float - total_asset_shares: float - last_update: int - config_asset_weight_init: float - config_asset_weight_maint: float - config_liability_weight_init: float - config_liability_weight_maint: float - config_deposit_limit: int - config_borrow_limit: int - config_interest_rate_config_optimal_utilization_rate: float - config_interest_rate_config_plateau_interest_rate: float - config_interest_rate_config_max_interest_rate: float - config_interest_rate_config_insurance_fee_fixed_apr: float - config_interest_rate_config_insurance_ir_fee: float - config_interest_rate_config_protocol_fixed_fee_apr: float - config_interest_rate_config_protocol_ir_fee: float - config_operational_state: str - config_oracle_setup: str - config_oracle_keys: str - config_risk_tier: str - - def __init__(self, parsed_data: NamedAccountData, account_update: "AccountUpdateRaw", idl_version: int): - super().__init__(parsed_data, account_update, idl_version) - - self.mint = str(parsed_data.data.mint) - self.mint_decimals = int(parsed_data.data.mint_decimals) - self.group = str(parsed_data.data.group) - self.asset_share_value = wrapped_i80f48_to_float(parsed_data.data.asset_share_value) - self.liability_share_value = wrapped_i80f48_to_float(parsed_data.data.liability_share_value) - self.liquidity_vault = str(parsed_data.data.liquidity_vault) - self.liquidity_vault_bump = int(parsed_data.data.liquidity_vault_bump) - self.liquidity_vault_authority_bump = int(parsed_data.data.liquidity_vault_authority_bump) - self.insurance_vault = str(parsed_data.data.insurance_vault) - self.insurance_vault_bump = int(parsed_data.data.insurance_vault_bump) - self.insurance_vault_authority_bump = int(parsed_data.data.insurance_vault_authority_bump) - self.fee_vault = str(parsed_data.data.fee_vault) - self.fee_vault_bump = int(parsed_data.data.fee_vault_bump) - self.fee_vault_authority_bump = int(parsed_data.data.fee_vault_authority_bump) - self.collected_insurance_fees_outstanding = wrapped_i80f48_to_float( - parsed_data.data.collected_insurance_fees_outstanding) - self.collected_group_fees_outstanding = wrapped_i80f48_to_float( - parsed_data.data.collected_group_fees_outstanding) - self.total_liability_shares = wrapped_i80f48_to_float(parsed_data.data.total_liability_shares) - self.total_asset_shares = wrapped_i80f48_to_float(parsed_data.data.total_asset_shares) - self.last_update = int(parsed_data.data.last_update) - - self.config_asset_weight_init = wrapped_i80f48_to_float(parsed_data.data.config.asset_weight_init) - self.config_asset_weight_maint = wrapped_i80f48_to_float(parsed_data.data.config.asset_weight_maint) - self.config_liability_weight_init = wrapped_i80f48_to_float(parsed_data.data.config.liability_weight_init) - self.config_liability_weight_maint = wrapped_i80f48_to_float(parsed_data.data.config.liability_weight_maint) - self.config_deposit_limit = int(parsed_data.data.config.deposit_limit) - self.config_borrow_limit = int(parsed_data.data.config.borrow_limit) - self.config_operational_state = enum_to_str(parsed_data.data.config.operational_state) - self.config_oracle_setup = enum_to_str(parsed_data.data.config.oracle_setup) - self.config_oracle_keys = str([str(pk) for pk in parsed_data.data.config.oracle_keys]) - self.config_risk_tier = enum_to_str(parsed_data.data.config.risk_tier) - - self.config_interest_rate_config_optimal_utilization_rate = wrapped_i80f48_to_float( - parsed_data.data.config.interest_rate_config.optimal_utilization_rate) - self.config_interest_rate_config_plateau_interest_rate = wrapped_i80f48_to_float( - parsed_data.data.config.interest_rate_config.plateau_interest_rate) - self.config_interest_rate_config_max_interest_rate = wrapped_i80f48_to_float( - parsed_data.data.config.interest_rate_config.max_interest_rate) - self.config_interest_rate_config_insurance_fee_fixed_apr = wrapped_i80f48_to_float( - parsed_data.data.config.interest_rate_config.insurance_fee_fixed_apr) - self.config_interest_rate_config_insurance_ir_fee = wrapped_i80f48_to_float( - parsed_data.data.config.interest_rate_config.insurance_ir_fee) - self.config_interest_rate_config_protocol_fixed_fee_apr = wrapped_i80f48_to_float( - parsed_data.data.config.interest_rate_config.protocol_fixed_fee_apr) - self.config_interest_rate_config_protocol_ir_fee = wrapped_i80f48_to_float( - parsed_data.data.config.interest_rate_config.protocol_ir_fee) - - -AccountUpdateRecordTypes = [MarginfiGroupUpdateRecord, - MarginfiAccountUpdateRecord, - LendingPoolBankUpdateRecord] - -AccountUpdateRecord = Union[ - MarginfiGroupUpdateRecord, - MarginfiAccountUpdateRecord, - LendingPoolBankUpdateRecord -] - -ACCOUNT_UPDATE_TO_RECORD_TYPE: Dict[str, Type[AccountUpdateRecord]] = { - f"{MARGINFI_GROUP_ACCOUNT_NAME}": MarginfiGroupUpdateRecord, - f"{MARGINFI_ACCOUNT_ACCOUNT_NAME}": MarginfiAccountUpdateRecord, - f"{LENDING_POOL_BANK_ACCOUNT_NAME}": LendingPoolBankUpdateRecord, -} diff --git a/observability/etl/dataflow-etls/dataflow_etls/orm/events.py b/observability/etl/dataflow-etls/dataflow_etls/orm/events.py deleted file mode 100644 index 115ca4db..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/orm/events.py +++ /dev/null @@ -1,473 +0,0 @@ -import uuid -from dataclasses import dataclass -from typing import Union, Optional, Dict, Type, TYPE_CHECKING -from anchorpy import Event, NamedInstruction - -from dataflow_etls.utils import pascal_to_snake_case, wrapped_i80f48_to_float, time_str, map_optional - -if TYPE_CHECKING: - from dataflow_etls.transaction_parsing import InstructionWithLogs - -# IDL event names -MARGINFI_GROUP_CREATE_EVENT_NAME = 'MarginfiGroupCreateEvent' -MARGINFI_GROUP_CONFIGURE_EVENT_NAME = 'MarginfiGroupConfigureEvent' -LENDING_POOL_BANK_CREATE_EVENT_NAME = 'LendingPoolBankCreateEvent' -LENDING_POOL_BANK_CONFIGURE_EVENT_NAME = 'LendingPoolBankConfigureEvent' -LENDING_POOL_BANK_ACCRUE_INTEREST_EVENT_NAME = 'LendingPoolBankAccrueInterestEvent' -LENDING_POOL_BANK_COLLECT_FEES_EVENT_NAME = 'LendingPoolBankCollectFeesEvent' -LENDING_POOL_BANK_HANDLE_BANKRUPTCY_EVENT_NAME = 'LendingPoolBankHandleBankruptcyEvent' -MARGINFI_ACCOUNT_CREATE_EVENT_NAME = 'MarginfiAccountCreateEvent' -LENDING_ACCOUNT_DEPOSIT_EVENT_NAME = 'LendingAccountDepositEvent' -LENDING_ACCOUNT_WITHDRAW_EVENT_NAME = 'LendingAccountWithdrawEvent' -LENDING_ACCOUNT_BORROW_EVENT_NAME = 'LendingAccountBorrowEvent' -LENDING_ACCOUNT_REPAY_EVENT_NAME = 'LendingAccountRepayEvent' -LENDING_ACCOUNT_LIQUIDATE_EVENT_NAME = 'LendingAccountLiquidateEvent' - - -@dataclass -class RecordBase: - SCHEMA = ",".join( - [ - "id:STRING", - "created_at:TIMESTAMP", - "idl_version:INTEGER", - "is_cpi:BOOLEAN", - "timestamp:TIMESTAMP", - "signature:STRING", - "indexing_address:STRING", - ] - ) - - id: str - created_at: str - idl_version: int - is_cpi: bool - # call_stack: List[str] - timestamp: str - signature: str - indexing_address: str - - def __init__(self, _event: Event, instruction: "InstructionWithLogs", _instruction_args: NamedInstruction): - self.id = str(uuid.uuid4()) - self.created_at = time_str() - self.timestamp = time_str(instruction.timestamp) - self.idl_version = instruction.idl_version - self.is_cpi = instruction.is_cpi - # self.call_stack=[str(pk) for pk in instruction.call_stack] - self.signature = instruction.signature - self.indexing_address = str(instruction.message.program_id) - - @classmethod - def get_tag(cls, snake_case: bool = False) -> str: - if snake_case: - return pascal_to_snake_case(cls.__name__) - else: - return cls.__name__ - - -# Event headers - -@dataclass -class AccountRecordBase(RecordBase): - SCHEMA = RecordBase.SCHEMA + "," + ",".join( - [ - "signer:STRING", - "marginfi_group:STRING", - "marginfi_account:STRING", - "marginfi_account_authority:STRING", - ] - ) - - signer: Optional[str] - marginfi_group: str - marginfi_account: str - marginfi_account_authority: str - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.signer = str(event.data.header.signer) if event.data.header.signer is not None else None - self.marginfi_group = str(event.data.header.marginfi_group) - self.marginfi_account = str(event.data.header.marginfi_account) - self.marginfi_account_authority = str(event.data.header.marginfi_account_authority) - - -@dataclass -class GroupRecordBase(RecordBase): - SCHEMA = RecordBase.SCHEMA + "," + ",".join( - [ - "signer:STRING", - "marginfi_group:STRING", - ] - ) - - signer: Optional[str] - marginfi_group: str - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.signer = str(event.data.header.signer) if event.data.header.signer is not None else None - self.marginfi_group = str(event.data.header.marginfi_group) - - -# Group events - - -@dataclass -class MarginfiGroupCreateRecord(GroupRecordBase): - SCHEMA = GroupRecordBase.SCHEMA - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - -@dataclass -class MarginfiGroupConfigureRecord(GroupRecordBase): - SCHEMA = GroupRecordBase.SCHEMA + "," + ",".join( - [ - "admin:STRING", - ] - ) - - admin: Optional[str] - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.admin = event.data.config.admin - - -@dataclass -class LendingPoolBankCreateRecord(GroupRecordBase): - SCHEMA = GroupRecordBase.SCHEMA + "," + ",".join( - [ - "bank:STRING", - "mint:STRING", - ] - ) - - bank: str - mint: str - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.bank = str(event.data.bank) - self.mint = str(event.data.mint) - - -@dataclass -class LendingPoolBankConfigureRecord(GroupRecordBase): - SCHEMA = GroupRecordBase.SCHEMA + "," + ",".join( - [ - "bank:STRING", - "mint:STRING", - "asset_weight_init:NUMERIC", - "asset_weight_maint:NUMERIC", - "liability_weight_init:NUMERIC", - "liability_weight_maint:NUMERIC", - "deposit_limit:BIGNUMERIC", - "borrow_limit:BIGNUMERIC", - "operational_state:STRING", - "oracle_setup:STRING", - "oracle_keys:STRING", - "optimal_utilization_rate:NUMERIC", - "plateau_interest_rate:NUMERIC", - "max_interest_rate:NUMERIC", - "insurance_fee_fixed_apr:NUMERIC", - "insurance_ir_fee:NUMERIC", - "protocol_fixed_fee_apr:NUMERIC", - "protocol_ir_fee:NUMERIC", - ] - ) - - bank: str - mint: str - - asset_weight_init: Optional[float] - asset_weight_maint: Optional[float] - - liability_weight_init: Optional[float] - liability_weight_maint: Optional[float] - - deposit_limit: Optional[int] - borrow_limit: Optional[int] - - operational_state: Optional[str] - oracle_setup: Optional[str] - oracle_keys: Optional[str] - - optimal_utilization_rate: Optional[float] - plateau_interest_rate: Optional[float] - max_interest_rate: Optional[float] - - insurance_fee_fixed_apr: Optional[float] - insurance_ir_fee: Optional[float] - protocol_fixed_fee_apr: Optional[float] - protocol_ir_fee: Optional[float] - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.bank = str(event.data.bank) - self.mint = str(event.data.mint) - - self.asset_weight_init = map_optional(event.data.config.asset_weight_init, wrapped_i80f48_to_float) - self.asset_weight_maint = map_optional(event.data.config.asset_weight_maint, wrapped_i80f48_to_float) - self.liability_weight_init = map_optional(event.data.config.liability_weight_init, wrapped_i80f48_to_float) - self.liability_weight_maint = map_optional(event.data.config.liability_weight_maint, wrapped_i80f48_to_float) - self.deposit_limit = event.data.config.deposit_limit - self.borrow_limit = event.data.config.borrow_limit - - self.operational_state = map_optional(event.data.config.operational_state, str) - if event.data.config.oracle: - self.oracle_setup = str(event.data.config.oracle.setup) - self.oracle_keys = str([str(pk) for pk in event.data.config.oracle.keys]) - else: - self.oracle_setup = None - self.oracle_keys = None - - self.optimal_utilization_rate = map_optional( - event.data.config.interest_rate_config.optimal_utilization_rate, wrapped_i80f48_to_float) - self.plateau_interest_rate = map_optional( - event.data.config.interest_rate_config.plateau_interest_rate, wrapped_i80f48_to_float) - self.max_interest_rate = map_optional( - event.data.config.interest_rate_config.max_interest_rate, wrapped_i80f48_to_float) - self.insurance_fee_fixed_apr = map_optional( - event.data.config.interest_rate_config.insurance_fee_fixed_apr, wrapped_i80f48_to_float) - self.insurance_ir_fee = map_optional( - event.data.config.interest_rate_config.insurance_ir_fee, wrapped_i80f48_to_float) - self.protocol_fixed_fee_apr = map_optional( - event.data.config.interest_rate_config.protocol_fixed_fee_apr, wrapped_i80f48_to_float) - self.protocol_ir_fee = map_optional( - event.data.config.interest_rate_config.protocol_ir_fee, wrapped_i80f48_to_float) - - -@dataclass -class LendingPoolBankAccrueInterestRecord(GroupRecordBase): - SCHEMA = GroupRecordBase.SCHEMA + "," + ",".join( - [ - "bank:STRING", - "mint:STRING", - "delta:BIGNUMERIC", - "fees_collected:BIGNUMERIC", - "insurance_collected:BIGNUMERIC", - ] - ) - - bank: str - mint: str - delta: int - fees_collected: float - insurance_collected: float - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.bank = str(event.data.bank) - self.mint = str(event.data.mint) - self.delta = event.data.delta - self.fees_collected = event.data.fees_collected - self.insurance_collected = event.data.insurance_collected - - -@dataclass -class LendingPoolBankCollectFeesRecord(GroupRecordBase): - SCHEMA = GroupRecordBase.SCHEMA + "," + ",".join( - [ - "bank:STRING", - "mint:STRING", - "group_fees_collected:BIGNUMERIC", - "group_fees_outstanding:BIGNUMERIC", - "insurance_fees_collected:BIGNUMERIC", - "insurance_fees_outstanding:BIGNUMERIC", - ] - ) - - bank: str - mint: str - group_fees_collected: float - group_fees_outstanding: float - insurance_fees_collected: float - insurance_fees_outstanding: float - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.bank = str(event.data.bank) - self.mint = str(event.data.mint) - self.group_fees_collected = event.data.group_fees_collected - self.group_fees_outstanding = event.data.group_fees_outstanding - self.insurance_fees_collected = event.data.insurance_fees_collected - self.insurance_fees_outstanding = event.data.insurance_fees_outstanding - - -@dataclass -class LendingPoolBankHandleBankruptcyRecord(GroupRecordBase): - SCHEMA = GroupRecordBase.SCHEMA + "," + ",".join( - [ - "bank:STRING", - "mint:STRING", - "bad_debt:BIGNUMERIC", - "covered_amount:BIGNUMERIC", - "socialized_amount:BIGNUMERIC", - ] - ) - - bank: str - mint: str - bad_debt: float - covered_amount: float - socialized_amount: float - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.bank = str(event.data.bank) - self.mint = str(event.data.mint) - self.bad_debt = event.data.bad_debt - self.covered_amount = event.data.covered_amount - self.socialized_amount = event.data.socialized_amount - - -# Account events - -@dataclass -class MarginfiAccountCreateRecord(AccountRecordBase): - SCHEMA = AccountRecordBase.SCHEMA - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - -@dataclass -class LendingAccountChangeLiquidityRecord(AccountRecordBase): - SCHEMA = AccountRecordBase.SCHEMA + "," + ",".join( - [ - "operation:STRING", - "bank:STRING", - "mint:STRING", - "amount:BIGNUMERIC", - "balance_closed:BOOLEAN" - ] - ) - - operation: str - bank: str - mint: str - amount: int - balance_closed: bool - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.operation = event.name.removeprefix("LendingAccount").removesuffix("Event").lower() - self.bank = str(event.data.bank) - self.mint = str(event.data.mint) - self.amount = event.data.amount - self.balance_closed = False - if event.name == LENDING_ACCOUNT_REPAY_EVENT_NAME or event.name == LENDING_ACCOUNT_WITHDRAW_EVENT_NAME: - self.balance_closed = event.data.close_balance - - -@dataclass -class LendingAccountLiquidateRecord(AccountRecordBase): - SCHEMA = AccountRecordBase.SCHEMA + "," + ",".join( - [ - "liquidatee_marginfi_account:STRING", - "liquidatee_marginfi_account_authority:STRING", - "asset_bank:STRING", - "asset_mint:STRING", - "liability_bank:STRING", - "liability_mint:STRING", - "liquidatee_pre_health:BIGNUMERIC", - "liquidatee_post_health:BIGNUMERIC", - "liquidatee_asset_pre_balance:BIGNUMERIC", - "liquidatee_liability_pre_balance:BIGNUMERIC", - "liquidator_asset_pre_balance:BIGNUMERIC", - "liquidator_liability_pre_balance:BIGNUMERIC", - "liquidatee_asset_post_balance:BIGNUMERIC", - "liquidatee_liability_post_balance:BIGNUMERIC", - "liquidator_asset_post_balance:BIGNUMERIC", - "liquidator_liability_post_balance:BIGNUMERIC", - ] - ) - - liquidatee_marginfi_account: str - liquidatee_marginfi_account_authority: str - asset_bank: str - asset_mint: str - liability_bank: str - liability_mint: str - liquidatee_pre_health: float - liquidatee_post_health: float - liquidatee_asset_pre_balance: float - liquidatee_liability_pre_balance: float - liquidator_asset_pre_balance: float - liquidator_liability_pre_balance: float - liquidatee_asset_post_balance: float - liquidatee_liability_post_balance: float - liquidator_asset_post_balance: float - liquidator_liability_post_balance: float - - def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): - super().__init__(event, instruction, instruction_args) - - self.liquidatee_marginfi_account = str(event.data.liquidatee_marginfi_account) - self.liquidatee_marginfi_account_authority = str(event.data.liquidatee_marginfi_account_authority) - self.asset_bank = str(event.data.asset_bank) - self.asset_mint = str(event.data.asset_mint) - self.liability_bank = str(event.data.liability_bank) - self.liability_mint = str(event.data.liability_mint) - self.liquidatee_pre_health = event.data.liquidatee_pre_health - self.liquidatee_post_health = event.data.liquidatee_post_health - self.liquidatee_asset_pre_balance = event.data.pre_balances.liquidatee_asset_balance - self.liquidatee_liability_pre_balance = event.data.pre_balances.liquidatee_liability_balance - self.liquidator_asset_pre_balance = event.data.pre_balances.liquidator_asset_balance - self.liquidator_liability_pre_balance = event.data.pre_balances.liquidator_liability_balance - self.liquidatee_asset_post_balance = event.data.post_balances.liquidatee_asset_balance - self.liquidatee_liability_post_balance = event.data.post_balances.liquidatee_liability_balance - self.liquidator_asset_post_balance = event.data.post_balances.liquidator_asset_balance - self.liquidator_liability_post_balance = event.data.post_balances.liquidator_liability_balance - - -EventRecordTypes = [MarginfiGroupCreateRecord, - MarginfiGroupConfigureRecord, - LendingPoolBankCreateRecord, - LendingPoolBankConfigureRecord, - LendingPoolBankAccrueInterestRecord, - LendingPoolBankCollectFeesRecord, - LendingPoolBankHandleBankruptcyRecord, - MarginfiAccountCreateRecord, - LendingAccountChangeLiquidityRecord, - LendingAccountLiquidateRecord] - -EventRecord = Union[ - MarginfiGroupCreateRecord, - MarginfiGroupConfigureRecord, - LendingPoolBankCreateRecord, - LendingPoolBankConfigureRecord, - LendingPoolBankAccrueInterestRecord, - LendingPoolBankCollectFeesRecord, - LendingPoolBankHandleBankruptcyRecord, - MarginfiAccountCreateRecord, - LendingAccountChangeLiquidityRecord, - LendingAccountLiquidateRecord -] - -EVENT_TO_RECORD_TYPE: Dict[str, Type[EventRecord]] = { - f"{MARGINFI_GROUP_CREATE_EVENT_NAME}": MarginfiGroupCreateRecord, - f"{MARGINFI_GROUP_CONFIGURE_EVENT_NAME}": MarginfiGroupConfigureRecord, - f"{LENDING_POOL_BANK_CREATE_EVENT_NAME}": LendingPoolBankCreateRecord, - f"{LENDING_POOL_BANK_CONFIGURE_EVENT_NAME}": LendingPoolBankConfigureRecord, - f"{LENDING_POOL_BANK_ACCRUE_INTEREST_EVENT_NAME}": LendingPoolBankAccrueInterestRecord, - f"{LENDING_POOL_BANK_COLLECT_FEES_EVENT_NAME}": LendingPoolBankCollectFeesRecord, - f"{LENDING_POOL_BANK_HANDLE_BANKRUPTCY_EVENT_NAME}": LendingPoolBankHandleBankruptcyRecord, - f"{MARGINFI_ACCOUNT_CREATE_EVENT_NAME}": MarginfiAccountCreateRecord, - f"{LENDING_ACCOUNT_DEPOSIT_EVENT_NAME}": LendingAccountChangeLiquidityRecord, - f"{LENDING_ACCOUNT_WITHDRAW_EVENT_NAME}": LendingAccountChangeLiquidityRecord, - f"{LENDING_ACCOUNT_BORROW_EVENT_NAME}": LendingAccountChangeLiquidityRecord, - f"{LENDING_ACCOUNT_REPAY_EVENT_NAME}": LendingAccountChangeLiquidityRecord, - f"{LENDING_ACCOUNT_LIQUIDATE_EVENT_NAME}": LendingAccountLiquidateRecord, -} diff --git a/observability/etl/dataflow-etls/dataflow_etls/transaction_parsing.py b/observability/etl/dataflow-etls/dataflow_etls/transaction_parsing.py deleted file mode 100644 index 81b9247c..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/transaction_parsing.py +++ /dev/null @@ -1,240 +0,0 @@ -import base64 -import json -import re -from dataclasses import dataclass, asdict -from datetime import datetime -from typing import List, Any, Callable, Optional, Dict, Sequence, Tuple, Generator, Union, TypedDict -from decimal import Decimal - -from anchorpy_core.idl import Idl -from solders.message import Message, MessageV0 -from based58 import based58 # type: ignore -from anchorpy import NamedInstruction -from solders.instruction import CompiledInstruction -from solders.pubkey import Pubkey -import apache_beam as beam # type: ignore - -from dataflow_etls.idl_versions import VersionedProgram, IdlPool, Cluster -from dataflow_etls.orm.events import EVENT_TO_RECORD_TYPE, EventRecord - - -@dataclass -class Instruction: - program_id: Pubkey - accounts: List[Pubkey] - data: bytes - - -@dataclass -class InstructionWithLogs: - timestamp: datetime - idl_version: int - signature: str - message: Instruction - logs: List[str] - inner_instructions: List["InstructionWithLogs"] - logs_truncated: bool - is_cpi: bool - # call_stack: List[Pubkey] - - -INVOKE_MESSAGE = "Program log: " -PROGRAM_LOG = "Program log: " -PROGRAM_DATA = "Program data: " -LOG_TRUNCATED = "Log truncated" - -TransactionRaw = TypedDict('TransactionRaw', { - 'id': str, - 'created_at': datetime, - 'timestamp': datetime, - 'signature': str, - 'indexing_address': str, - 'slot': Decimal, - 'signer': str, - 'success': bool, - 'version': str, - 'fee': Decimal, - 'meta': str, - 'message': str, -}) - - -class IndexedProgramNotSupported(Exception): - pass - - -def extract_events_from_tx(tx: TransactionRaw, min_idl_version: int, cluster: Cluster, idl_pool: IdlPool) -> List[ - EventRecord]: - indexed_program_id_str = tx["indexing_address"] - indexed_program_id = Pubkey.from_string(indexed_program_id_str) - tx_slot = int(tx["slot"]) - - try: - idl_raw, idl_version = idl_pool.get_idl_for_slot(indexed_program_id_str, tx_slot) - except KeyError: - raise IndexedProgramNotSupported(f"Unsupported indexed program {indexed_program_id_str}") - - idl = Idl.from_json(idl_raw) - program = VersionedProgram(cluster, idl_version, idl, indexed_program_id) - - if min_idl_version is not None and idl_version < min_idl_version: - return [] - - meta = json.loads(tx["meta"]) - message_bytes = base64.b64decode(tx["message"]) - - tx_version = tx["version"] - message_decoded: Union[Message, MessageV0] - if tx_version == "legacy": - message_decoded = Message.from_bytes(message_bytes) - elif tx_version == "0": - message_decoded = MessageV0.from_bytes(message_bytes[1:]) - else: - return [] - - merged_instructions = merge_instructions_and_cpis(message_decoded.instructions, meta["innerInstructions"]) - expanded_instructions = expand_instructions(message_decoded.account_keys, merged_instructions) - ixs_with_logs = reconcile_instruction_logs(tx["timestamp"], tx["signature"], expanded_instructions, - meta["logMessages"], idl_version) - - records_list = [] - for ix_with_logs in ixs_with_logs: - records_list.extend(extract_events_from_ix(ix_with_logs, program)) - - return records_list - - -def merge_instructions_and_cpis(message_instructions: List[CompiledInstruction], inner_instructions: List[Any]) -> List[ - CompiledInstruction]: - def search(array: List[Any], callback: Callable[[Any], bool]) -> Optional[int]: - for i, elem in enumerate(array): - if callback(elem): - return i - return None - - compiled_instructions: List[CompiledInstruction] = [] - for ix_index, instruction in enumerate(message_instructions): - compiled_instructions.append(instruction) - inner_ixs_index = search(inner_instructions, lambda inner_ixs: bool(inner_ixs["index"] == ix_index)) - if inner_ixs_index is not None: - for ix_raw in inner_instructions[inner_ixs_index]["instructions"]: - compiled_instructions.append( - CompiledInstruction(program_id_index=ix_raw["programIdIndex"], accounts=bytes(ix_raw["accounts"]), - data=based58.b58decode(str.encode(ix_raw["data"])))) - - return compiled_instructions - - -def expand_instructions(account_keys: List[Pubkey], compiled_instructions: List[CompiledInstruction]) -> List[ - Instruction]: - expanded_instructions = [] - for ix in compiled_instructions: - expanded_instruction = Instruction(data=ix.data, - accounts=[account_keys[account_index] for account_index in ix.accounts], - program_id=account_keys[ix.program_id_index]) - expanded_instructions.append(expanded_instruction) - return expanded_instructions - - -def reconcile_instruction_logs(timestamp: datetime, signature: str, instructions: List[Instruction], logs: List[str], - idl_version: int) -> \ - List[InstructionWithLogs]: - depth = 0 - instructions_consumed = 0 - instructions_with_logs: List[InstructionWithLogs] = [] - - for log in logs: - if log.startswith(LOG_TRUNCATED): - ix = get_latest_ix_ref(instructions_with_logs, depth) - ix.logs_truncated = True - else: - invoke_regex = r"Program (?P\w+) invoke" - matches = re.search(invoke_regex, log) - if matches is not None: - target_instruction_list = instructions_with_logs - for i in range(depth): - target_instruction_list = target_instruction_list[-1].inner_instructions - - message = instructions[instructions_consumed] - target_instruction_list.append( - InstructionWithLogs(timestamp=timestamp, idl_version=idl_version, signature=signature, logs=[log], - message=message, - inner_instructions=[], logs_truncated=False, is_cpi=(depth > 0))) - depth += 1 - instructions_consumed += 1 - else: - if "success" in log or "failed" in log: - ix = get_latest_ix_ref(instructions_with_logs, depth) - ix.logs.append(log) - depth -= 1 - else: - ix = get_latest_ix_ref(instructions_with_logs, depth) - ix.logs.append(log) - - return instructions_with_logs - - -def get_latest_ix_ref(instructions: List[InstructionWithLogs], stack_depth: int) -> "InstructionWithLogs": - target_instruction_list = instructions - for i in range(stack_depth - 1): - target_instruction_list = target_instruction_list[-1].inner_instructions - return target_instruction_list[-1] - - -def extract_events_from_ix(ix: InstructionWithLogs, program: VersionedProgram) -> List[EventRecord]: - ix_events: List[EventRecord] = [] - - if ix.message.program_id == program.program_id: - ix_events.extend(create_records_from_ix(ix, program)) - - for inner_ix in ix.inner_instructions: - ix_events.extend(extract_events_from_ix(inner_ix, program)) - - return ix_events - - -def create_records_from_ix(ix: InstructionWithLogs, program: VersionedProgram) -> Sequence[EventRecord]: - records: List[EventRecord] = [] - - try: - parsed_ix: NamedInstruction = program.coder.instruction.parse(ix.message.data) - except Exception as e: - print(f"failed to parse instruction data in tx {ix.signature} ({ix.timestamp})", e) - return records - - for log in ix.logs: - if not log.startswith(PROGRAM_DATA): - continue - - event_encoded = log[len(PROGRAM_DATA):] - try: - event_bytes = base64.b64decode(event_encoded) - except Exception as e: - print(f"error: failed to decode base64 event string in tx {ix.signature}", e) - continue - - try: - event = program.coder.events.parse(event_bytes) - except Exception as e: - print(f"failed to parse event in tx {ix.signature}", e) - continue - - if event is None or event.name not in EVENT_TO_RECORD_TYPE: - print(f"discarding unsupported event in tx {ix.signature}") - print(event) - else: - # noinspection PyPep8Naming - RecordType = EVENT_TO_RECORD_TYPE[event.name] - records.append(RecordType(event, ix, parsed_ix)) - - return records - - -class DispatchEventsDoFn(beam.DoFn): # type: ignore - def process(self, record: EventRecord, *args: Tuple[Any], **kwargs: Dict[str, Tuple[Any]]) -> Generator[ - str, None, None]: - yield beam.pvalue.TaggedOutput(record.get_tag(), record) - - -def dictionify_record(record: EventRecord) -> Dict[str, Any]: - return asdict(record) diff --git a/observability/etl/dataflow-etls/dataflow_etls/utils.py b/observability/etl/dataflow-etls/dataflow_etls/utils.py deleted file mode 100644 index 8d44c156..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/utils.py +++ /dev/null @@ -1,40 +0,0 @@ -import re -from datetime import datetime, timezone -from typing import NamedTuple, TypeVar, Optional, Callable, Any - -from decimal import Decimal - - -def pascal_to_snake_case(string: str) -> str: - return re.sub('(?!^)([A-Z]+)', r'_\1', string).lower() - - -WrappedI80F48 = NamedTuple('WrappedI80F48', [('value', int)]) - - -def wrapped_i80f48_to_float(wrapped_i80f48: WrappedI80F48) -> float: - nb_of_fractional_bits = 48 - value = Decimal(wrapped_i80f48.value) - value = value / 2 ** nb_of_fractional_bits - return float(value) - - -def enum_to_str(enum: Any) -> str: - return enum.__class__.__name__.lower() - - -InputType = TypeVar('InputType') -OutputType = TypeVar('OutputType') - - -def map_optional(element: Optional[InputType], fn: Callable[[InputType], OutputType]) -> Optional[OutputType]: - if element is not None: - return fn(element) - else: - return None - - -def time_str(dt: Optional[datetime] = None) -> str: - if dt is None: - dt = datetime.now(timezone.utc) - return dt.strftime("%Y-%m-%d %H:%M:%S %Z") diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/job.py b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/job.py deleted file mode 100644 index 6f33d274..00000000 --- a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/job.py +++ /dev/null @@ -1,141 +0,0 @@ -import argparse -import logging -from typing import List, Optional, Any, Dict, Union -import apache_beam as beam # type: ignore -from apache_beam.options.pipeline_options import PipelineOptions # type: ignore - -from dataflow_etls.account_parsing import parse_account, OwnerProgramNotSupported, dictionify_record, DispatchEventsDoFn -from dataflow_etls.idl_versions import Cluster, IdlPool -from dataflow_etls.orm.accounts import AccountUpdateRecordTypes, AccountUpdateRecord - - -def run( - input_table: str, - output_table_namespace: str, - cluster: Cluster, - min_idl_version: int, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - beam_args: Optional[List[str]] = None, -) -> None: - if beam_args is None: - beam_args = [] - - idl_pool = IdlPool(cluster) - - def parse_account_internal(tx: Any) -> List[AccountUpdateRecord]: - try: - return parse_account(tx, min_idl_version, cluster, idl_pool) - except OwnerProgramNotSupported: - return [] - - """Build and run the pipeline.""" - pipeline_options = PipelineOptions(beam_args, save_main_session=True) - - if start_date is not None and end_date is not None: - input_query = f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}" AND DATE(timestamp) < "{end_date}"' - elif start_date is not None: - input_query = ( - f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}"' - ) - elif end_date is not None: - input_query = ( - f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) < "{end_date}"' - ) - else: - input_query = f"SELECT * FROM `{input_table}`" - - with beam.Pipeline(options=pipeline_options) as pipeline: - # Define steps - read_raw_txs = beam.io.ReadFromBigQuery(query=input_query, use_standard_sql=True) - - extract_events = beam.FlatMap(parse_account_internal) - - dispatch_events = beam.ParDo(DispatchEventsDoFn()).with_outputs( - *[rt.get_tag() for rt in AccountUpdateRecordTypes]) - - dictionify_events = beam.Map(dictionify_record) - - writers: Dict[str, Union[beam.io.WriteToText, beam.io.WriteToBigQuery]] = {} - for rt in AccountUpdateRecordTypes: - if output_table_namespace == "local_file": # For testing purposes - writers[rt.get_tag()] = beam.io.WriteToText(f"account_updates_{rt.get_tag(snake_case=True)}") - else: - writers[rt.get_tag()] = beam.io.WriteToBigQuery( - f"{output_table_namespace}_{rt.get_tag(snake_case=True)}", - schema=rt.SCHEMA, - write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND, - create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, - ) - - # Define pipeline - tagged_events = ( - pipeline - | "ReadRawTxs" >> read_raw_txs - | "ExtractEvents" >> extract_events - | "DispatchEvents" >> dispatch_events - ) - - for rt in AccountUpdateRecordTypes: - (tagged_events[rt.get_tag()] - | f"Dictionify{rt.get_tag()}" >> dictionify_events - | f"Write{rt.get_tag()}" >> writers[rt.get_tag()] - ) - - -def main() -> None: - logging.getLogger().setLevel(logging.INFO) - - parser = argparse.ArgumentParser() - parser.add_argument( - "--input_table", - type=str, - required=True, - help="Input BigQuery table specified as: " - "PROJECT.DATASET.TABLE.", - ) - parser.add_argument( - "--output_table_namespace", - type=str, - required=True, - help="Output BigQuery namespace where parsed account tables are located: PROJECT:DATASET.TABLE", - ) - parser.add_argument( - "--cluster", - type=str, - required=False, - default="mainnet", - help="Solana cluster being indexed: mainnet | devnet", - ) - parser.add_argument( - "--min_idl_version", - type=int, - required=False, - default=0, - help="Minimum IDL version to consider: int", - ) - parser.add_argument( - "--start_date", - type=str, - help="Start date to consider (inclusive) as: YYYY-MM-DD", - ) - parser.add_argument( - "--end_date", - type=str, - help="End date to consider (exclusive) as: YYYY-MM-DD", - ) - known_args, remaining_args = parser.parse_known_args() - - run( - input_table=known_args.input_table, - output_table_namespace=known_args.output_table_namespace, - cluster=known_args.cluster, - min_idl_version=known_args.min_idl_version, - start_date=known_args.start_date, - end_date=known_args.end_date, - beam_args=remaining_args, - ) - - -if __name__ == "__main__": - main() diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/metadata.json b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/metadata.json deleted file mode 100644 index 5a82d4b2..00000000 --- a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/metadata.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "marginfi-v2-account-parsing-batch", - "description": "Parses individual account updates from a BigQuery table and stores them in dedicated BigQuery tables.", - "parameters": [ - { - "name": "input_table", - "label": "BigQuery input table name.", - "helpText": "Name of the input table to consume from.", - "regexes": [ - "([^.]+.)?[^.]+[.].+" - ] - }, - { - "name": "output_table_namespace", - "label": "BigQuery output tables namespace.", - "helpText": "Namespace where the BigQuery output tables are located.", - "regexes": [ - "([^:]+:)?[^.]+[.].+" - ] - }, - { - "name": "cluster", - "label": "Solana cluster.", - "isOptional": true, - "helpText": "Cluster where the account updates are executed (used to pull IDL version depending on tx slot).", - "regexes": ["mainnet|devnet"] - }, - { - "name": "min_idl_version", - "label": "Minimum IDL version.", - "isOptional": true, - "helpText": "Minimum IDL version for which txs will be parsed. Default: 0", - "regexes": [] - }, - { - "name": "start_date", - "label": "Start date.", - "isOptional": true, - "helpText": "Start date to consider (inclusive).", - "regexes": [] - }, - { - "name": "end_date", - "label": "End date.", - "isOptional": true, - "helpText": "End date to consider (exclusive).", - "regexes": [] - } - ] -} diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/job.py b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/job.py deleted file mode 100644 index f1a926bc..00000000 --- a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/job.py +++ /dev/null @@ -1,146 +0,0 @@ -import argparse -from dateutil import parser -import json -import logging -from typing import List, Optional, Any, Dict, Union -from decimal import Decimal - -import apache_beam as beam # type: ignore -from apache_beam.options.pipeline_options import PipelineOptions # type: ignore - -from dataflow_etls.account_parsing import parse_account, OwnerProgramNotSupported, dictionify_record, \ - DispatchEventsDoFn, AccountUpdateRaw -from dataflow_etls.idl_versions import Cluster, IdlPool -from dataflow_etls.orm.accounts import AccountUpdateRecordTypes, AccountUpdateRecord - - -def parse_json(message: bytes) -> AccountUpdateRaw: - account_update_raw = json.loads(message.decode("utf-8")) - return AccountUpdateRaw( - id=account_update_raw['id'], - created_at=parser.parse(account_update_raw['created_at']), - timestamp=parser.parse(account_update_raw['timestamp']), - owner=account_update_raw['owner'], - slot=Decimal(account_update_raw['slot']), - pubkey=account_update_raw['pubkey'], - txn_signature=account_update_raw['txn_signature'], - lamports=Decimal(account_update_raw['lamports']), - executable=bool(account_update_raw['executable']), - rent_epoch=Decimal(account_update_raw['rent_epoch']), - data=account_update_raw['data'], - ) - - -def run( - input_topic: str, - input_subscription: str, - output_table_namespace: str, - cluster: Cluster, - min_idl_version: int, - beam_args: Optional[List[str]] = None, -) -> None: - if beam_args is None: - beam_args = [] - - idl_pool = IdlPool(cluster) - - def parse_account_internal(tx: Any) -> List[AccountUpdateRecord]: - try: - return parse_account(tx, min_idl_version, cluster, idl_pool) - except OwnerProgramNotSupported: - return [] - - """Build and run the pipeline.""" - pipeline_options = PipelineOptions(beam_args, save_main_session=True, streaming=True) - - with beam.Pipeline(options=pipeline_options) as pipeline: - # Define steps - read_raw_txs = beam.io.ReadFromPubSub( - topic=input_topic, subscription=input_subscription - ).with_output_types(bytes) - - parse_to_raw_txs = beam.Map(parse_json) - - extract_events = beam.FlatMap(parse_account_internal) - - dispatch_events = beam.ParDo(DispatchEventsDoFn()).with_outputs( - *[rt.get_tag() for rt in AccountUpdateRecordTypes]) - - dictionify_events = beam.Map(dictionify_record) - - writers: Dict[str, Union[beam.io.WriteToText, beam.io.WriteToBigQuery]] = {} - for rt in AccountUpdateRecordTypes: - if output_table_namespace == "local_file": # For testing purposes - writers[rt.get_tag()] = beam.io.WriteToText(f"account_updates_{rt.get_tag(snake_case=True)}") - else: - writers[rt.get_tag()] = beam.io.WriteToBigQuery( - f"{output_table_namespace}_{rt.get_tag(snake_case=True)}", - schema=rt.SCHEMA, - write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND, - create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, - ) - - # Define pipeline - tagged_events = ( - pipeline - | "ReadRawTxs" >> read_raw_txs - | "ParseTxsToRawTxs" >> parse_to_raw_txs - | "ExtractEvents" >> extract_events - | "DispatchEvents" >> dispatch_events - ) - - for rt in AccountUpdateRecordTypes: - (tagged_events[rt.get_tag()] - | f"Dictionify{rt.get_tag()}" >> dictionify_events - | f"Write{rt.get_tag()}" >> writers[rt.get_tag()] - ) - - -def main() -> None: - logging.getLogger().setLevel(logging.INFO) - - parser = argparse.ArgumentParser() - parser.add_argument( - "--input_topic", - type=str, - help='Input PubSub topic of the form "projects//topics/."', - ) - parser.add_argument( - "--input_subscription", - type=str, - help='Input PubSub subscription of the form "projects//subscriptions/."', - ) - parser.add_argument( - "--output_table_namespace", - type=str, - required=True, - help="Output BigQuery namespace where parsed account tables are located: PROJECT:DATASET.TABLE", - ) - parser.add_argument( - "--cluster", - type=str, - required=False, - default="mainnet", - help="Solana cluster being indexed: mainnet | devnet", - ) - parser.add_argument( - "--min_idl_version", - type=int, - required=False, - default=0, - help="Minimum IDL version to consider: int", - ) - known_args, remaining_args = parser.parse_known_args() - - run( - input_topic=known_args.input_topic, - input_subscription=known_args.input_subscription, - output_table_namespace=known_args.output_table_namespace, - cluster=known_args.cluster, - min_idl_version=known_args.min_idl_version, - beam_args=remaining_args, - ) - - -if __name__ == "__main__": - main() diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/metadata.json b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/metadata.json deleted file mode 100644 index ed2ce4bc..00000000 --- a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/metadata.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "name": "marginfi-v2-account-parsing-stream", - "description": "Parses individual account updates from a Pub/Sub topic/subscription and stores them in dedicated BigQuery tables.", - "parameters": [ - { - "name": "input_topic", - "label": "Input PubSub topic.", - "isOptional": true, - "helpText": "Name of the input PubSub topic to consume from.", - "regexes": [ - "projects/[^/]+/topics/[a-zA-Z][-_.~+%a-zA-Z0-9]{2,}" - ] - }, - { - "name": "input_subscription", - "label": "Input PubSub subscription.", - "isOptional": true, - "helpText": "Name of the input PubSub subscription to consume from.", - "regexes": [ - "projects/[^/]+/subscriptions/[a-zA-Z][-_.~+%a-zA-Z0-9]{2,}" - ] - }, - { - "name": "output_table_namespace", - "label": "BigQuery output tables namespace.", - "helpText": "Namespace where the BigQuery output tables are located.", - "regexes": [ - "([^:]+:)?[^.]+[.].+" - ] - }, - { - "name": "cluster", - "label": "Solana cluster.", - "isOptional": true, - "helpText": "Cluster where the account updates are executed (used to pull IDL version depending on tx slot).", - "regexes": ["mainnet|devnet"] - }, - { - "name": "min_idl_version", - "label": "Minimum IDL version.", - "isOptional": true, - "helpText": "Minimum IDL version for which txs will be parsed. Default: 0", - "regexes": [] - } - ] -} diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/job.py b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/job.py deleted file mode 100644 index 4c24d8ee..00000000 --- a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/job.py +++ /dev/null @@ -1,162 +0,0 @@ -import argparse -import logging -from typing import List, Optional, Union, Any, Dict -import apache_beam as beam # type: ignore -from apache_beam.options.pipeline_options import PipelineOptions # type: ignore - -from dataflow_etls.orm.events import EventRecordTypes, EventRecord -from dataflow_etls.idl_versions import Cluster, IdlPool -from dataflow_etls.transaction_parsing import dictionify_record, DispatchEventsDoFn, extract_events_from_tx, \ - IndexedProgramNotSupported - - -def run( - input_table: str, - output_table_namespace: str, - cluster: Cluster, - min_idl_version: int, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - start_timestamp: Optional[str] = None, - end_timestamp: Optional[str] = None, - beam_args: Optional[List[str]] = None, -) -> None: - if beam_args is None: - beam_args = [] - - idl_pool = IdlPool(cluster) - - def extract_events_from_tx_internal(tx: Any) -> List[EventRecord]: - try: - return extract_events_from_tx(tx, min_idl_version, cluster, idl_pool) - except IndexedProgramNotSupported: - return [] - - """Build and run the pipeline.""" - pipeline_options = PipelineOptions(beam_args, save_main_session=True) - - if start_date is not None and end_date is not None: - input_query = f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}" AND DATE(timestamp) < "{end_date}"' - elif start_timestamp is not None and end_timestamp is not None: - input_query = f'SELECT * FROM `{input_table}` WHERE timestamp >= "{start_timestamp}" AND timestamp < "{end_timestamp}"' - elif start_date is not None: - input_query = ( - f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}"' - ) - elif end_date is not None: - input_query = ( - f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) < "{end_date}"' - ) - elif end_timestamp is not None: - input_query = ( - f'SELECT * FROM `{input_table}` WHERE timestamp < "{end_timestamp}"' - ) - print("yes", input_query) - else: - input_query = f"SELECT * FROM `{input_table}`" - - with beam.Pipeline(options=pipeline_options) as pipeline: - # Define steps - read_raw_txs = beam.io.ReadFromBigQuery(query=input_query, use_standard_sql=True) - - extract_events = beam.FlatMap(extract_events_from_tx_internal) - - dispatch_events = beam.ParDo(DispatchEventsDoFn()).with_outputs(*[rt.get_tag() for rt in EventRecordTypes]) - - dictionify_events = beam.Map(dictionify_record) - - writers: Dict[str, Union[beam.io.WriteToText, beam.io.WriteToBigQuery]] = {} - for rt in EventRecordTypes: - if output_table_namespace == "local_file": # For testing purposes - writers[rt.get_tag()] = beam.io.WriteToText(f"events_{rt.get_tag(snake_case=True)}") - else: - writers[rt.get_tag()] = beam.io.WriteToBigQuery( - f"{output_table_namespace}_{rt.get_tag(snake_case=True)}", - schema=rt.SCHEMA, - write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND, - create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, - ) - - # Define pipeline - tagged_events = ( - pipeline - | "ReadRawTxs" >> read_raw_txs - | "ExtractEvents" >> extract_events - | "DispatchEvents" >> dispatch_events - ) - - for rt in EventRecordTypes: - (tagged_events[rt.get_tag()] - | f"Dictionify{rt.get_tag()}" >> dictionify_events - | f"Write{rt.get_tag()}" >> writers[rt.get_tag()] - ) - - -def main() -> None: - logging.getLogger().setLevel(logging.INFO) - - parser = argparse.ArgumentParser() - parser.add_argument( - "--input_table", - type=str, - required=True, - help="Input BigQuery table specified as: " - "PROJECT.DATASET.TABLE.", - ) - parser.add_argument( - "--output_table_namespace", - type=str, - required=True, - help="Output BigQuery namespace where event tables are located: PROJECT:DATASET.TABLE", - ) - parser.add_argument( - "--cluster", - type=str, - required=False, - default="mainnet", - help="Solana cluster being indexed: mainnet | devnet", - ) - parser.add_argument( - "--min_idl_version", - type=int, - required=False, - default=0, - help="Minimum IDL version to consider: int", - ) - parser.add_argument( - "--start_date", - type=str, - help="Start date to consider (inclusive) as: YYYY-MM-DD", - ) - parser.add_argument( - "--end_date", - type=str, - help="End date to consider (exclusive) as: YYYY-MM-DD", - ) - parser.add_argument( - "--start_timestamp", - type=str, - help="Start timestamp to consider (inclusive) as: YYYY-MM-DD HH:MM:SS", - ) - parser.add_argument( - "--end_timestamp", - type=str, - help="End timestamp to consider (exclusive) as: YYYY-MM-DD HH:MM:SS", - ) - known_args, remaining_args = parser.parse_known_args() - - run( - input_table=known_args.input_table, - output_table_namespace=known_args.output_table_namespace, - cluster=known_args.cluster, - min_idl_version=known_args.min_idl_version, - start_date=known_args.start_date, - end_date=known_args.end_date, - start_timestamp=known_args.start_timestamp, - end_timestamp=known_args.end_timestamp, - beam_args=remaining_args, - ) - - -if __name__ == "__main__": - main() diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/metadata.json b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/metadata.json deleted file mode 100644 index 4bac544d..00000000 --- a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/metadata.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "marginfi-v2-event-parsing-batch", - "description": "Parses individual raw transactions from a BigQuery table and stores them in dedicated BigQuery tables.", - "parameters": [ - { - "name": "input_table", - "label": "BigQuery input table name.", - "helpText": "Name of the input table to consume from.", - "regexes": [ - "([^.]+.)?[^.]+[.].+" - ] - }, - { - "name": "output_table_namespace", - "label": "BigQuery output tables namespace.", - "helpText": "Namespace where the BigQuery output tables are located.", - "regexes": [ - "([^:]+:)?[^.]+[.].+" - ] - }, - { - "name": "cluster", - "label": "Solana cluster.", - "isOptional": true, - "helpText": "Cluster where the processed txs are executed (used to pull IDL version depending on tx slot).", - "regexes": ["mainnet|devnet"] - }, - { - "name": "min_idl_version", - "label": "Minimum IDL version.", - "isOptional": true, - "helpText": "Minimum IDL version for which txs will be parsed. Default: 0", - "regexes": [] - }, - { - "name": "start_date", - "label": "Start date.", - "isOptional": true, - "helpText": "Start date to consider (inclusive).", - "regexes": [] - }, - { - "name": "end_date", - "label": "End date.", - "isOptional": true, - "helpText": "End date to consider (exclusive).", - "regexes": [] - } - ] -} diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/job.py b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/job.py deleted file mode 100644 index afbbec20..00000000 --- a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/job.py +++ /dev/null @@ -1,156 +0,0 @@ -import argparse -from dateutil import parser -import json -import logging -from typing import List, Optional, Union, Any, Dict -from decimal import Decimal - -import apache_beam as beam # type: ignore -from apache_beam.options.pipeline_options import PipelineOptions # type: ignore - -from dataflow_etls.orm.events import EventRecord, EventRecordTypes -from dataflow_etls.idl_versions import Cluster, IdlPool -from dataflow_etls.transaction_parsing import extract_events_from_tx, dictionify_record, DispatchEventsDoFn, \ - TransactionRaw, IndexedProgramNotSupported - - -def parse_json(message: bytes) -> TransactionRaw: - tx_raw = json.loads(message.decode("utf-8")) - return TransactionRaw( - id=tx_raw["id"], - created_at=parser.parse(tx_raw['created_at']), - timestamp=parser.parse(tx_raw['timestamp']), - signature=tx_raw['signature'], - indexing_address=tx_raw['indexing_address'], - slot=Decimal(tx_raw['slot']), - signer=tx_raw['signer'], - success=bool(tx_raw['success']), - version=tx_raw['version'], - fee=Decimal(tx_raw['fee']), - meta=tx_raw['meta'], - message=tx_raw['message'], - ) - - -def run( - input_topic: str, - input_subscription: str, - output_table_namespace: str, - cluster: Cluster, - min_idl_version: int, - # window_interval_sec: int = 60, - beam_args: Optional[List[str]] = None, -) -> None: - if beam_args is None: - beam_args = [] - - idl_pool = IdlPool(cluster) - - def extract_events_from_tx_internal(tx: Any) -> List[EventRecord]: - try: - return extract_events_from_tx(tx, min_idl_version, cluster, idl_pool) - except IndexedProgramNotSupported: - return [] - - """Build and run the pipeline.""" - pipeline_options = PipelineOptions(beam_args, save_main_session=True, streaming=True) - - with beam.Pipeline(options=pipeline_options) as pipeline: - # Define steps - read_raw_txs = beam.io.ReadFromPubSub( - topic=input_topic, subscription=input_subscription - ).with_output_types(bytes) - - parse_to_raw_txs = beam.Map(parse_json) - - # group_in_windows = beam.WindowInto(window.FixedWindows(window_interval_sec, 0)) - - extract_events = beam.FlatMap(extract_events_from_tx_internal) - - dispatch_events = beam.ParDo(DispatchEventsDoFn()).with_outputs(*[rt.get_tag() for rt in EventRecordTypes]) - - dictionify_events = beam.Map(dictionify_record) - - writers: Dict[str, Union[beam.io.WriteToText, beam.io.WriteToBigQuery]] = {} - for rt in EventRecordTypes: - if output_table_namespace == "local_file": # For testing purposes - writers[rt.get_tag()] = beam.io.WriteToText(f"events_{rt.get_tag(snake_case=True)}") - else: - writers[rt.get_tag()] = beam.io.WriteToBigQuery( - f"{output_table_namespace}_{rt.get_tag(snake_case=True)}", - schema=rt.SCHEMA, - write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND, - create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, - ) - - # Define pipeline - tagged_events = ( - pipeline - | "ReadRawTxs" >> read_raw_txs - | "ParseTxsToRawTxs" >> parse_to_raw_txs - | "ExtractEvents" >> extract_events - | "DispatchEvents" >> dispatch_events - ) - - for rt in EventRecordTypes: - (tagged_events[rt.get_tag()] - | f"Dictionify{rt.get_tag()}" >> dictionify_events - | f"Write{rt.get_tag()}" >> writers[rt.get_tag()] - ) - - -def main() -> None: - logging.getLogger().setLevel(logging.INFO) - - parser = argparse.ArgumentParser() - parser.add_argument( - "--input_topic", - type=str, - help='Input PubSub topic of the form "projects//topics/."', - ) - parser.add_argument( - "--input_subscription", - type=str, - help='Input PubSub subscription of the form "projects//subscriptions/."', - ) - parser.add_argument( - "--output_table_namespace", - type=str, - required=True, - help="Output BigQuery namespace where event tables are located: PROJECT:DATASET.TABLE", - ) - parser.add_argument( - "--window_interval_sec", - default=60, - type=int, - help="Window interval in seconds for grouping incoming messages.", - ) - parser.add_argument( - "--cluster", - type=str, - required=False, - default="mainnet", - help="Solana cluster being indexed: mainnet | devnet", - ) - parser.add_argument( - "--min_idl_version", - type=int, - required=False, - default=0, - help="Minimum IDL version to consider: int", - ) - known_args, remaining_args = parser.parse_known_args() - - run( - input_topic=known_args.input_topic, - input_subscription=known_args.input_subscription, - output_table_namespace=known_args.output_table_namespace, - # window_interval_sec=known_args.window_interval_sec, - cluster=known_args.cluster, - min_idl_version=known_args.min_idl_version, - beam_args=remaining_args, - ) - - -if __name__ == "__main__": - main() diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/metadata.json b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/metadata.json deleted file mode 100644 index 252e77a6..00000000 --- a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/metadata.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "name": "marginfi-v2-event-parsing-stream", - "description": "Parses individual raw transactions from a Pub/Sub topic/subscription and stores them in dedicated BigQuery tables.", - "parameters": [ - { - "name": "input_topic", - "label": "Input PubSub topic.", - "isOptional": true, - "helpText": "Name of the input PubSub topic to consume from.", - "regexes": [ - "projects/[^/]+/topics/[a-zA-Z][-_.~+%a-zA-Z0-9]{2,}" - ] - }, - { - "name": "input_subscription", - "label": "Input PubSub subscription.", - "isOptional": true, - "helpText": "Name of the input PubSub subscription to consume from.", - "regexes": [ - "projects/[^/]+/subscriptions/[a-zA-Z][-_.~+%a-zA-Z0-9]{2,}" - ] - }, - { - "name": "output_table_namespace", - "label": "BigQuery output tables namespace.", - "helpText": "Namespace where the BigQuery output tables are located.", - "regexes": [ - "([^:]+:)?[^.]+[.].+" - ] - }, - { - "name": "cluster", - "label": "Solana cluster.", - "isOptional": true, - "helpText": "Cluster where the processed txs are executed (used to pull IDL version depending on tx slot).", - "regexes": ["mainnet|devnet"] - }, - { - "name": "min_idl_version", - "label": "Minimum IDL version.", - "isOptional": true, - "helpText": "Minimum IDL version for which txs will be parsed. Default: 0", - "regexes": [] - } - ] -} diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/update_running_pipeline b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/update_running_pipeline deleted file mode 100644 index 3aecd96a..00000000 --- a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/update_running_pipeline +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -e - -job_id=$1 - -[ -z "$job_id" ] && echo "Missing job_id argument" && exit 1 - -gcloud dataflow flex-template run "$job_id" \ - --template-file-gcs-location gs://dataflow_jobs_marginfi_v2/templates/marginfi-v2-event-parsing-stream.json \ - --region us-central1 \ - --parameters input_topic=projects/marginfi-dev/topics/marginfi-v2-mainnet-transactions-raw,output_table_namespace=marginfi-dev:marginfi_v2.event,cluster=mainnet \ - --update \ No newline at end of file diff --git a/observability/etl/dataflow-etls/poetry.toml b/observability/etl/dataflow-etls/poetry.toml deleted file mode 100644 index 62e2dff2..00000000 --- a/observability/etl/dataflow-etls/poetry.toml +++ /dev/null @@ -1,3 +0,0 @@ -[virtualenvs] -in-project = true -create = true diff --git a/observability/etl/dataflow-etls/pyproject.toml b/observability/etl/dataflow-etls/pyproject.toml deleted file mode 100644 index a6bfee77..00000000 --- a/observability/etl/dataflow-etls/pyproject.toml +++ /dev/null @@ -1,53 +0,0 @@ -[tool.poetry] -name = "dataflow_etls" -version = "0.1.0" -description = "" -authors = ["man0s <95379755+losman0s@users.noreply.github.com>"] - -[tool.poetry.scripts] -job = "dataflow_etls.job:main" - -[tool.poetry.dependencies] -python = "^3.9" -wheel = "^0.38.4" -apache-beam = { extras = ["gcp"], version = "2.44.0" } -solders = "^0.14.0" -anchorpy = "^0.16.0" -based58 = "^0.1.1" -isort = "^5.12.0" - -[tool.poetry.group.dev.dependencies] -typer = "^0.7.0" -ipython = "^8.10.0" -autoflake = "^2.0.1" -genpy = "^2022.1" -solana = "^0.29.1" -pre-commit = "^2.18.1" -mypy = "^0.950" - -[tool.black] -line-length = 132 -target-version = ['py37', 'py38', 'py39', 'py310' ] -include = '(src\/scripts\/.*$|\.pyi?$)' -exclude = ''' -/( - \.git - | __pycache__ - | \.tox - | \.venv - | \.poetry - | build - | dist - | docs - | notes -)/ -''' - -[tool.isort] -profile = "black" -line_length = 132 -skip_glob = [ "docs", "notes" ] - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/observability/etl/dataflow-etls/scripts/build_job_template b/observability/etl/dataflow-etls/scripts/build_job_template deleted file mode 100755 index f4dd9e90..00000000 --- a/observability/etl/dataflow-etls/scripts/build_job_template +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -e - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -ROOT_DIR=$(realpath "${SCRIPT_DIR}/../") - -job_dir=$1 -image_version=$2 - -[ -z "$job_dir" ] && echo "Missing job_dir argument" && exit 1 -[ -z "$image_version" ] && echo "Missing image_version argument" && exit 1 - - -job_dir=$(realpath "$job_dir") -job_name=$(basename "$job_dir") -job_dir_rel=$(realpath --relative-to="$ROOT_DIR" "$job_dir") - -cd "$ROOT_DIR" - -local_image_name="$job_name" -gcp_image_name="us-east1-docker.pkg.dev/marginfi-dev/main/dataflow/$local_image_name" - -docker build --platform=linux/amd64 --build-arg JOB_DIRECTORY="$job_dir_rel" --tag "$job_name" "$ROOT_DIR" -docker tag "$local_image_name" "$gcp_image_name:$image_version" diff --git a/observability/etl/dataflow-etls/scripts/generate_sample_events.sh b/observability/etl/dataflow-etls/scripts/generate_sample_events.sh deleted file mode 100755 index d56e201b..00000000 --- a/observability/etl/dataflow-etls/scripts/generate_sample_events.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash -set -e - -PROGRAM_ID=5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK - -random_id=$(echo $RANDOM | md5sum | head -c 20; echo) -test_profile="test-devnet-$random_id" - -RPC_ENDPOINT=https://devnet.rpcpool.com/ -KEYPAIR_PATH=~/.config/solana/id.json - -echo "-> Creating test profile $test_profile" -mfi profile create \ - --cluster devnet \ - --name "$test_profile" \ - --keypair-path "$KEYPAIR_PATH" \ - --rpc-url "$RPC_ENDPOINT" \ - --program-id "$PROGRAM_ID" - -mfi profile set "$test_profile" - -echo "-> Admin creates marginfi group" -mfi group create -y - -echo "-> Admin updates marginfi group" -mfi group update -y - -echo "-> Admin creates USDC bank" -usdc_bank=$(mfi group add-bank \ - --mint F9jRT1xL7PCRepBuey5cQG5vWHFSbnvdWxJWKqtzMDsd \ - --asset-weight-init 0.85 \ - --asset-weight-maint 0.9 \ - --liability-weight-maint 1.1 \ - --liability-weight-init 1.15 \ - --deposit-limit 1000000000000000\ - --borrow-limit 1000000000000000\ - --pyth-oracle 5SSkXsEKQepHHAewytPVwdej4epN1nxgLVM84L4KXgy7 \ - --optimal-utilization-rate 0.9 \ - --plateau-interest-rate 1 \ - --max-interest-rate 10 \ - --insurance-fee-fixed-apr 0.01 \ - --insurance-ir-fee 0.1 \ - --protocol-fixed-fee-apr 0.01 \ - --protocol-ir-fee 0.1 \ - --risk-tier collateral \ - -y) -echo "USDC bank: $usdc_bank" - -echo "-> Admin creates SOL bank" -sol_bank=$(mfi group add-bank \ - --mint 4Bn9Wn1sgaD5KfMRZjxwKFcrUy6NKdyqLPtzddazYc4x \ - --asset-weight-init 0.75 \ - --asset-weight-maint 0.8 \ - --liability-weight-maint 1.2 \ - --liability-weight-init 1.25 \ - --deposit-limit 1000000000000000\ - --borrow-limit 1000000000000000\ - --pyth-oracle J83w4HKfqxwcq3BEMMkPFSppX3gqekLyLJBexebFVkix \ - --optimal-utilization-rate 0.8 \ - --plateau-interest-rate 1 \ - --max-interest-rate 20 \ - --insurance-fee-fixed-apr 0.01 \ - --insurance-ir-fee 0.1 \ - --protocol-fixed-fee-apr 0.01 \ - --protocol-ir-fee 0.1 \ - --risk-tier collateral \ - -y) -echo "SOL bank: $sol_bank" - -echo "-> Admin configures SOL bank" -mfi bank update "$sol_bank" \ - --asset-weight-init 1 \ - --asset-weight-maint 1 \ - -y - -echo "-> Admin configures USDC bank" -mfi bank update "$usdc_bank" \ - --asset-weight-init 1 \ - --asset-weight-maint 1 \ - -y - -mfi account create -y - -echo "-> Random user lends USDC" -mfi account deposit "$usdc_bank" 0.01 -y - -echo "-> Liquidatee creates new mfi account" -liquidatee_account=$(mfi account create -y) -echo "liquidatee account: $liquidatee_account" - -echo "-> Liquidatee deposits SOL" -mfi account deposit "$sol_bank" 0.01 -y - -echo "-> Liquidatee borrows USDC" -mfi account borrow "$usdc_bank" 0.01 -y - -echo "-> Admin triggers bad health by setting SOL asset weights to 0" -mfi bank update "$sol_bank" \ - --asset-weight-init 0 \ - --asset-weight-maint 0 \ - -y - -echo "-> Liquidator creates mfi account" -mfi account create -y - -echo "-> Liquidator deposits USDC to pay off liquidatee's debt" -mfi account deposit "$usdc_bank" 0.01 -y - -echo "-> Liquidator liquidates liquidatee for half its assets" -mfi account liquidate \ - --liquidatee-marginfi-account="$liquidatee_account" \ - --asset-bank="$sol_bank" \ - --liability-bank="$usdc_bank" \ - --ui-asset-amount=0.0001 \ - -y - -echo "-> Admin handles remainder of bad debt through handle bankruptcy" -mfi bank handle-bankruptcy \ - --bank="$usdc_bank" \ - --marginfi-account="$liquidatee_account" \ - -y - -echo "-> Admin collects fees on USDC bank" -mfi bank collect-fees \ - --bank="$usdc_bank" \ - -y - -echo "-> Admin collects fees on SOL bank" -mfi bank collect-fees \ - --bank="$sol_bank" \ - -y diff --git a/observability/etl/dataflow-etls/scripts/lint b/observability/etl/dataflow-etls/scripts/lint deleted file mode 100755 index fa697155..00000000 --- a/observability/etl/dataflow-etls/scripts/lint +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -poetry run mypy . \ No newline at end of file diff --git a/observability/etl/dataflow-etls/scripts/playground.py b/observability/etl/dataflow-etls/scripts/playground.py deleted file mode 100644 index 35c1fab0..00000000 --- a/observability/etl/dataflow-etls/scripts/playground.py +++ /dev/null @@ -1,111 +0,0 @@ -from pathlib import Path -from pprint import pprint -import based58 -from anchorpy import Program -from anchorpy_core.idl import Idl -from solana.rpc.api import Client -from solders.pubkey import Pubkey -from solders.signature import Signature - -from decimal import Decimal - - -def wrapped_i80f48_to_float(wrapped_i80f48_value: int): - nb_of_fractional_bits = 48 - value = Decimal(wrapped_i80f48_value) - value = value / 2 ** nb_of_fractional_bits - return value - - -print(wrapped_i80f48_to_float(-54953626681867088)) - -sample_logs = ["Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [1]", "Program log: CreateIdempotent", - "Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 7338 of 400000 compute units", - "Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success", - "Program A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4 invoke [1]", - "Program log: Instruction: LendingAccountWithdraw", - "Program log: Withdrawing all: 1000000 of F9jRT1xL7PCRepBuey5cQG5vWHFSbnvdWxJWKqtzMDsd in GhV6ZftLXv3o38CHMhX6nu8GkxS3kvrHSSCVpGFTysUC", - "Program log: withdraw_spl_transfer: amount: 1000000 from J9SAzLYETfcXBdrvswRaNUiGaMtmLiucwEJKEFW8d3FA to 4U3UNQU7spMKzY1cUviRdj9zAT2cVbGQEzioey1mCCZM, auth Fx99GAAXXk43peMfHxS2S7xTubazffA5h7ftmTEJK2bk", - "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]", "Program log: Instruction: Transfer", - "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4645 of 315437 compute units", - "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", - "Program data: A9yU8yH5NlgFAAAAMC4xLjAF3hNv9VErtBxgns6TItDZs+IZ/Y6TNWGuTe16nnh4ye4CH6/bWLrtvuQMibfybmkmRSqFhHygh92ElC0GqUlcO8hKNv2sSk0p0XfYfozLqjspIR2sHsbx0eDvsV5tGY7pPk3uIRtapASqm9ALTdv++zxMXXSinbwKl99MTnuDkdJAOsgK1DjHp0u5LCmsGj4g7ioWbEEtiXn/B9aQ3U+4QEIPAAAAAAAB", - "Program log: Expecting 0 remaining accounts", "Program log: Got 0 remaining accounts", - "Program log: check_health: assets 0 - liabs: 0", - "Program A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4 consumed 88164 of 392662 compute units", - "Program A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4 success"] - -sample_inner_ixs = [ - {"index": 1, "instructions": [{"programIdIndex": 9, "accounts": [5, 1, 4], - "data": "11115hc6izQ5YEyLuqy666n8aGeujSQofY7ibwnRv64oCXeYpCg8t6ZaiuSwwbw6ev76Ut"}]}] - -sample_message = "gAEABgwF3hNv9VErtBxgns6TItDZs+IZ/Y6TNWGuTe16nnh4yTOAiZ9axygRNRuzZCiQK4q5zvPfaH0CnR4tEUdeu2zY7gIfr9tYuu2+5AyJt/JuaSZFKoWEfKCH3YSULQapSVzpPk3uIRtapASqm9ALTdv++zxMXXSinbwKl99MTnuDkd4jxKWJrU3oI9SPceEA9wtTBJ5T78IpGTiq1XRzIrqD/r/JWnhhuWaJpdf6l+yCA38eN/l9xqMSjBXsYMAOea2MlyWPTiSJ8bs9ECkUjg2DC1oTmdr/EIQEjnvY2+n4WdJAOsgK1DjHp0u5LCmsGj4g7ioWbEEtiXn/B9aQ3U+4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG3fbh12Whk9nL4UbO63msHLSF7V9bN5E6jPWFfv8AqYd/H/1F6tUC+2l0WY9HS4WATM+2LA2i9Tq46cjeMS2lO8hKNv2sSk0p0XfYfozLqjspIR2sHsbx0eDvsV5tGY4/gcRh2keI2GQribtQSPXEZxYR1wZZ93XFhp/+f3fIFAIGBgABAAcICQEBCggLAgADAQQFCRIkSEoT0tLAwEBCDwAAAAAAAQEA" - -http_client = Client("https://api.devnet.solana.com") -tx = http_client.get_transaction( - Signature.from_string("4WdMP6s8SasqWrWpscurrN1r8ueLZNSdq81nLRJfyhsDU7xMz47t8xYh4hTvTy2C2sZ8Frc3x8urjfkYwKTg7dNT"), - max_supported_transaction_version=0) - -message = tx.value.transaction.transaction.message -pprint(message) -ix_data = message.instructions[0].data -# message_bytes = base64.b64decode(cpi.data) -# message_decoded = MessageV0.from_bytes(message_bytes[1:]) - -# merged_instructions = merge_instructions_and_cpis(message_decoded.instructions, sample_inner_ixs) -# expanded_instructions = expand_instructions(message_decoded.account_keys, merged_instructions) -# ix_with_logs = reconcile_instruction_logs("zrozUUvTujLSCxyT7JHtX48V5MYgkdQi4FAh5HreaH7p8n93bCDCo1huJsVBYiBkNXvFij7QgqYFC5jRRcXxpzi", expanded_instructions, sample_logs) - -path = Path("idls/devnet/marginfi-v2.json") -raw = path.read_text() -idl = Idl.from_json(raw) - -program = Program(idl, Pubkey.from_string("A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4")) -print(ix_data) -# message_bytes = based58.b58decode(b"CkHi86f1tcCVWQEX8TFsY7AwRnzu5ZQsh8Jnva7t4euknF6qyvtJuFDkiNsaraQBtLZcUNRCPJhAwexoH9EbTGFGi6uWVukVidkGz3LTaeevMSN6uqj5xjvVrDZVA5buqKQz86uErcvki7RNvez7QeoaBc19PV4YcLSfHkZpWUGxCCW87cfsShqCJgrsdjW45gkfKxqxb7t8T8FwVpudj2v7hZJzBqHSVvnQKWHW5ENmHHbYfSAFUGjQUhct6AnAWsoJ5XWoAUrm8G3ppN3fJe8vMGCvPfEnz76ea9LYQDeYSCqKg9f8QwZ2jj8z7xNfBUJw3MJh2hSxWxL635Hrx2xKnRpFT7vNugx2fpwwasGfBYkfMivjVfTVKcjJWSK46NXzmqhKX16ct7vpeBasue8eUM9hAtG5KDqs8XXQ3QkqeChm6qX2GJ7ohY6TRZAoQid767qLY84ZHEutqUibTQCtRUT15hHwbRRBAmAStWeBKJDopDUyHvxRXMxsG7dddT4pqNEukFRNu1chj4Sn2k2D8j9gehTESuKxtD6KVKBD3zb1MwpXGJK41TkUtTrDfn81REFmcAATB5srpXdfVig9XFDkZa2CUNrXwmmgMwR2LNx5Fv1mUtXwr93cvULv1VeXc5Qmfy66LK1mDaFVtR5iGqKAD4Tjf7vwJTA4i1Q1EFHrcZfFS3YT3QqVbbnjDGj59rnDtLqoWqzTLMSdnekbLnU98rndUgc8XUL3EX") -message_bytes = based58.b58decode(str.encode(str(ix_data))) -parsed = program.coder.instruction.parse(message_bytes) -print(parsed) - -# for ix in ix_with_logs: -# print("\n\n=============================================") -# print("pid:", ix.message.program_id) -# # print(log) -# if ix.message.program_id == program.program_id: -# parsed = program.coder.instruction.parse(ix.message.data) -# print(parsed) -# -# for inner_ix in ix.inner_instructions: -# print("\n CPI <<<<") -# print("pid:", inner_ix.message.program_id) -# if inner_ix.message.program_id == program.program_id: -# parsed = program.coder.instruction.parse(inner_ix.message.data) -# print(parsed) - -# events_coder: EventCoder = EventCoder(idl) -# -# parser = EventParser(program.program_id, program.coder) -# events = [] -# parser.parse_logs(sample_log, lambda evt: print(evt)) - -# event = MarginfiAccountCreateEvent(header=AccountEventHeader( -# version="0.1.0", -# signer=Pubkey.from_string("ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL"), -# marginfi_account=Pubkey.from_string("ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL"), -# marginfi_group=Pubkey.from_string("ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL"), -# )) -# -# pprint.pprint(str(event.__dataclass_fields__)) - -# test = bytes([3, 3, 3]) -# test1 = base64.b64encode(test) -# test2 = base64.b64decode(test1) -# print(test2) -# -# ix_coder = program.coder.instruction -# message = "" -# sample_message_bytes = base64.b64decode(message) -# print(sample_message_bytes) -# sample_message = MessageV0.from_bytes(sample_message_bytes[1:]) -# print(sample_message) -# message_decoded = ix_coder.parse(bytes) diff --git a/observability/etl/dataflow-etls/scripts/sync_job_template b/observability/etl/dataflow-etls/scripts/sync_job_template deleted file mode 100755 index 2cd6126c..00000000 --- a/observability/etl/dataflow-etls/scripts/sync_job_template +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -e - -job_dir=$1 -image_version=$2 - -[ -z "$job_dir" ] && echo "Missing job_dir argument" && exit 1 -[ -z "$image_version" ] && echo "Missing image_version argument" && exit 1 - -job_dir=$(realpath "$job_dir") -job_name=$(basename "$job_dir") - -local_image_name="$job_name" -gcp_image_name="us-east1-docker.pkg.dev/marginfi-dev/main/dataflow/$local_image_name" -gcp_template_gs_path="gs://dataflow_jobs_marginfi_v2/templates/$job_name.json" -metadata_local_path="$job_dir/metadata.json" - -gcloud dataflow flex-template build \ - "$gcp_template_gs_path" \ - --image "$gcp_image_name:$image_version" \ - --sdk-language "PYTHON" \ - --metadata-file "$metadata_local_path" diff --git a/observability/etl/dataflow-etls/scripts/upload_job_template b/observability/etl/dataflow-etls/scripts/upload_job_template deleted file mode 100755 index 5475179c..00000000 --- a/observability/etl/dataflow-etls/scripts/upload_job_template +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -e - -job_dir=$1 -image_version=$2 - -[ -z "$job_dir" ] && echo "Missing job_dir argument" && exit 1 -[ -z "$image_version" ] && echo "Missing image_version argument" && exit 1 - -job_dir=$(realpath "$job_dir") -job_name=$(basename "$job_dir") - -local_image_name="$job_name" -gcp_image_name="us-east1-docker.pkg.dev/marginfi-dev/main/dataflow/$local_image_name" - -docker push "$gcp_image_name:$image_version" diff --git a/observability/etl/dataflow-etls/setup.py b/observability/etl/dataflow-etls/setup.py deleted file mode 100644 index 5b626d13..00000000 --- a/observability/etl/dataflow-etls/setup.py +++ /dev/null @@ -1,14 +0,0 @@ -import setuptools - -setuptools.setup( - name='dataflow_etls', - version='0.1.0', - install_requires=[ - 'anchorpy==0.16.0', - 'apache-beam[gcp]==2.45.0', - "based58==0.1.1", - "solders==0.14.4 ", - ], - packages=setuptools.find_packages(), - include_package_data=True -) diff --git a/observability/indexer/build.rs b/observability/indexer/build.rs deleted file mode 100644 index 8a74cf06..00000000 --- a/observability/indexer/build.rs +++ /dev/null @@ -1,18 +0,0 @@ -fn main() -> anyhow::Result<()> { - compile_protos()?; - Ok(()) -} - -fn compile_protos() -> anyhow::Result<()> { - std::env::set_var("PROTOC", protobuf_src::protoc()); - tonic_build::configure() - .protoc_arg("--experimental_allow_proto3_optional") - .type_attribute("PubsubTransaction", "#[derive(serde::Serialize)]") - .type_attribute("PubsubAccountUpdate", "#[derive(serde::Serialize)]") - .compile( - &["protos/geyser.proto", "protos/gcp_pubsub.proto"], - &["protos"], - ) - .unwrap(); - Ok(()) -} diff --git a/observability/indexer/protos/gcp_pubsub.proto b/observability/indexer/protos/gcp_pubsub.proto deleted file mode 100644 index f725265c..00000000 --- a/observability/indexer/protos/gcp_pubsub.proto +++ /dev/null @@ -1,33 +0,0 @@ -syntax = "proto2"; - -package gcp_pubsub; - -message PubsubTransaction { - required string id = 1; - required string created_at = 2; - required string timestamp = 3; - required string signature = 4; - required string indexing_address = 5; - required uint64 slot = 6; - required string signer = 7; - required bool success = 8; - required string version = 9; - required uint64 fee = 10; - required string meta = 11; - required string message = 12; -} - -message PubsubAccountUpdate { - required string id = 1; - required string created_at = 2; - required string timestamp = 3; - required string owner = 4; - required uint64 slot = 5; - required string pubkey = 6; - optional string txn_signature = 7; - optional uint64 write_version = 8; - required uint64 lamports = 9; - required bool executable = 10; - required uint64 rent_epoch = 11; - required string data = 12; -} diff --git a/observability/indexer/protos/geyser.proto b/observability/indexer/protos/geyser.proto deleted file mode 100644 index 84d22d59..00000000 --- a/observability/indexer/protos/geyser.proto +++ /dev/null @@ -1,95 +0,0 @@ -syntax = "proto3"; - -import public "solana-storage-v1.10.40.proto"; - -package geyser; - -service Geyser { - rpc Subscribe(stream SubscribeRequest) returns (stream SubscribeUpdate) {} -} - -message SubscribeRequest { - map accounts = 1; - map slots = 2; - map transactions = 3; - map blocks = 4; -} - -message SubscribeRequestFilterAccounts { - repeated string account = 2; - repeated string owner = 3; -} - -message SubscribeRequestFilterSlots {} - -message SubscribeRequestFilterTransactions { - optional bool vote = 1; - optional bool failed = 2; - repeated string account_include = 3; - repeated string account_exclude = 4; -} - -message SubscribeRequestFilterBlocks {} - -message SubscribeUpdate { - repeated string filters = 1; - oneof update_oneof { - SubscribeUpdateAccount account = 2; - SubscribeUpdateSlot slot = 3; - SubscribeUpdateTransaction transaction = 4; - SubscribeUpdateBlock block = 5; - SubscribeUpdatePing ping = 6; - } -} - -message SubscribeUpdateAccount { - SubscribeUpdateAccountInfo account = 1; - uint64 slot = 2; - bool is_startup = 3; -} - -message SubscribeUpdateAccountInfo { - bytes pubkey = 1; - uint64 lamports = 2; - bytes owner = 3; - bool executable = 4; - uint64 rent_epoch = 5; - bytes data = 6; - uint64 write_version = 7; - optional bytes txn_signature = 8; -} - -message SubscribeUpdateSlot { - uint64 slot = 1; - optional uint64 parent = 2; - SubscribeUpdateSlotStatus status = 3; -} - -enum SubscribeUpdateSlotStatus { - PROCESSED = 0; - CONFIRMED = 1; - FINALIZED = 2; -} - -message SubscribeUpdateTransaction { - SubscribeUpdateTransactionInfo transaction = 1; - uint64 slot = 2; -} - -message SubscribeUpdateTransactionInfo { - bytes signature = 1; - bool is_vote = 2; - solana.storage.ConfirmedBlock.Transaction transaction = 3; - solana.storage.ConfirmedBlock.TransactionStatusMeta meta = 4; - // uint64 index = 5; -} - -message SubscribeUpdateBlock { - uint64 slot = 1; - string blockhash = 2; - solana.storage.ConfirmedBlock.Rewards rewards = 3; - solana.storage.ConfirmedBlock.UnixTimestamp block_time = 4; - solana.storage.ConfirmedBlock.BlockHeight block_height = 5; -} - -message SubscribeUpdatePing {} \ No newline at end of file diff --git a/observability/indexer/protos/solana-storage-v1.10.40.proto b/observability/indexer/protos/solana-storage-v1.10.40.proto deleted file mode 100644 index 2be3e7a0..00000000 --- a/observability/indexer/protos/solana-storage-v1.10.40.proto +++ /dev/null @@ -1,118 +0,0 @@ -syntax = "proto3"; - -package solana.storage.ConfirmedBlock; - -message ConfirmedBlock { - string previous_blockhash = 1; - string blockhash = 2; - uint64 parent_slot = 3; - repeated ConfirmedTransaction transactions = 4; - repeated Reward rewards = 5; - UnixTimestamp block_time = 6; - BlockHeight block_height = 7; -} - -message ConfirmedTransaction { - Transaction transaction = 1; - TransactionStatusMeta meta = 2; -} - -message Transaction { - repeated bytes signatures = 1; - Message message = 2; -} - -message Message { - MessageHeader header = 1; - repeated bytes account_keys = 2; - bytes recent_blockhash = 3; - repeated CompiledInstruction instructions = 4; - bool versioned = 5; - repeated MessageAddressTableLookup address_table_lookups = 6; -} - -message MessageHeader { - uint32 num_required_signatures = 1; - uint32 num_readonly_signed_accounts = 2; - uint32 num_readonly_unsigned_accounts = 3; -} - -message MessageAddressTableLookup { - bytes account_key = 1; - bytes writable_indexes = 2; - bytes readonly_indexes = 3; -} - -message TransactionStatusMeta { - TransactionError err = 1; - uint64 fee = 2; - repeated uint64 pre_balances = 3; - repeated uint64 post_balances = 4; - repeated InnerInstructions inner_instructions = 5; - bool inner_instructions_none = 10; - repeated string log_messages = 6; - bool log_messages_none = 11; - repeated TokenBalance pre_token_balances = 7; - repeated TokenBalance post_token_balances = 8; - repeated Reward rewards = 9; - repeated bytes loaded_writable_addresses = 12; - repeated bytes loaded_readonly_addresses = 13; -} - -message TransactionError { - bytes err = 1; -} - -message InnerInstructions { - uint32 index = 1; - repeated CompiledInstruction instructions = 2; -} - -message CompiledInstruction { - uint32 program_id_index = 1; - bytes accounts = 2; - bytes data = 3; -} - -message TokenBalance { - uint32 account_index = 1; - string mint = 2; - UiTokenAmount ui_token_amount = 3; - string owner = 4; - string program_id = 5; -} - -message UiTokenAmount { - double ui_amount = 1; - uint32 decimals = 2; - string amount = 3; - string ui_amount_string = 4; -} - -enum RewardType { - Unspecified = 0; - Fee = 1; - Rent = 2; - Staking = 3; - Voting = 4; -} - -message Reward { - string pubkey = 1; - int64 lamports = 2; - uint64 post_balance = 3; - RewardType reward_type = 4; - string commission = 5; -} - -message Rewards { - repeated Reward rewards = 1; -} - -message UnixTimestamp { - int64 timestamp = 1; -} - -message BlockHeight { - uint64 block_height = 1; -} \ No newline at end of file diff --git a/observability/indexer/src/commands/backfill.rs b/observability/indexer/src/commands/backfill.rs deleted file mode 100644 index 049ed769..00000000 --- a/observability/indexer/src/commands/backfill.rs +++ /dev/null @@ -1,190 +0,0 @@ -use crate::{ - common::Target, - utils::{ - big_query::DATE_FORMAT_STR, - protos::gcp_pubsub, - transactions_crawler::{ - TransactionsCrawler, TransactionsCrawlerConfig, TransactionsCrawlerContext, - }, - }, -}; -use anyhow::Result; -use base64::{engine::general_purpose, Engine}; -use chrono::{NaiveDateTime, Utc}; -use envconfig::Envconfig; -use futures::future::join_all; -use google_cloud_default::WithAuthExt; -use google_cloud_googleapis::pubsub::v1::PubsubMessage; -use google_cloud_pubsub::client::{Client, ClientConfig}; -use itertools::Itertools; -use solana_sdk::{pubkey::Pubkey, signature::Signature, transaction::TransactionVersion}; -use std::{str::FromStr, sync::Arc, time::Duration}; -use tracing::error; -use uuid::Uuid; - -#[derive(Envconfig, Debug, Clone)] -pub struct BackfillConfig { - #[envconfig(from = "BACKFILL_RPC_ENDPOINT")] - pub rpc_endpoint: String, - #[envconfig(from = "BACKFILL_SIGNATURE_FETCH_LIMIT")] - pub signature_fetch_limit: usize, - #[envconfig(from = "BACKFILL_MAX_CONCURRENT_REQUESTS")] - pub max_concurrent_requests: usize, - #[envconfig(from = "BACKFILL_MAX_PENDING_SIGNATURES")] - pub max_pending_signatures: usize, - #[envconfig(from = "BACKFILL_MONITOR_INTERVAL")] - pub monitor_interval: u64, - #[envconfig(from = "BACKFILL_PROGRAM_ID")] - pub program_id: Pubkey, - #[envconfig(from = "BACKFILL_BEFORE_SIGNATURE")] - pub before: Option, - #[envconfig(from = "BACKFILL_UNTIL_SIGNATURE")] - pub until: Option, - - #[envconfig(from = "BACKFILL_PROJECT_ID")] - pub project_id: String, - #[envconfig(from = "BACKFILL_PUBSUB_TOPIC_NAME")] - pub topic_name: String, - #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] - pub gcp_sa_key: String, -} - -pub async fn backfill(config: BackfillConfig) -> Result<()> { - let config_clone = config.clone(); - - let tx_crawler = TransactionsCrawler::new_with_config(TransactionsCrawlerConfig { - rpc_endpoint: config_clone.rpc_endpoint, - signature_fetch_limit: config_clone.signature_fetch_limit, - max_concurrent_requests: config_clone.max_concurrent_requests, - max_pending_signatures: config_clone.max_pending_signatures, - monitor_interval: config_clone.monitor_interval, - targets: [Target { - before: config_clone - .before - .map(|sig_str| Signature::from_str(&sig_str).unwrap()), - until: config_clone - .until - .map(|sig_str| Signature::from_str(&sig_str).unwrap()), - address: config_clone.program_id, - }] - .to_vec(), - }); - - let transaction_processor = |ctx: Arc| async move { - push_transactions_to_pubsub(ctx, config).await.unwrap() - }; - - tx_crawler.run_async(&transaction_processor).await.unwrap(); - - Ok(()) -} - -pub async fn push_transactions_to_pubsub( - ctx: Arc, - config: BackfillConfig, -) -> Result<()> { - let topic_name = config.topic_name.as_str(); - - let client_config = ClientConfig::default().with_auth().await?; - let client = Client::new(client_config).await?; - - let topic = client.topic(topic_name); - topic - .exists(None, None) - .await - .unwrap_or_else(|_| panic!("topic {} not found", topic_name)); - - let publisher = topic.new_publisher(None); - - loop { - let mut transactions_data = vec![]; - { - let signatures_queue = ctx.transaction_queue.lock().unwrap(); - while !signatures_queue.is_empty() { - transactions_data.push(signatures_queue.pop().unwrap()); - } - } - if transactions_data.is_empty() { - tokio::time::sleep(Duration::from_millis(10)).await; - continue; - } - - let mut messages = vec![]; - - transactions_data.iter().for_each(|transaction_data| { - let now = Utc::now(); - - let tx_with_meta = &transaction_data.transaction.transaction; - let tx_decoded = tx_with_meta.transaction.decode().unwrap(); - - // println!( - // "{:?} - {}", - // transaction_data.indexing_address, - // tx_decoded.signatures.first().unwrap().to_string(), - // ); - - let message = gcp_pubsub::PubsubTransaction { - id: Uuid::new_v4().to_string(), - created_at: now.format(DATE_FORMAT_STR).to_string(), - timestamp: NaiveDateTime::from_timestamp_opt( - transaction_data.transaction.block_time.unwrap_or(0), - 0, - ) - .unwrap() - .format(DATE_FORMAT_STR) - .to_string(), - signature: tx_decoded.signatures.first().unwrap().to_string(), - indexing_address: transaction_data.indexing_address.to_string(), - slot: transaction_data.transaction.slot, - signer: tx_decoded - .message - .static_account_keys() - .first() - .unwrap() - .to_string(), - success: tx_with_meta.meta.clone().unwrap().err.is_none(), - version: tx_with_meta - .version - .clone() - .map(|v| match v { - TransactionVersion::Legacy(_) => "legacy".to_string(), - TransactionVersion::Number(version) => version.to_string(), - }) - .unwrap_or_else(|| "unknown".to_string()), - fee: tx_with_meta.meta.clone().map(|meta| meta.fee).unwrap_or(0), - meta: tx_with_meta - .meta - .clone() - .map(|meta| serde_json::to_string(&meta).unwrap()) - .unwrap(), - message: general_purpose::STANDARD.encode(tx_decoded.message.serialize()), - }; - - let message_str = serde_json::to_string(&message).unwrap(); - let message_bytes = message_str.as_bytes().to_vec(); - messages.push(PubsubMessage { - data: message_bytes.into(), - ..PubsubMessage::default() - }); - }); - - // Send a message. There are also `publish_bulk` and `publish_immediately` methods. - let awaiters = publisher.publish_bulk(messages).await; - - // The get method blocks until a server-generated ID or an error is returned for the published message. - let pub_results = join_all( - awaiters - .into_iter() - .map(|awaiter| awaiter.get(None)) - .collect_vec(), - ) - .await; - - pub_results.into_iter().for_each(|result| match result { - Ok(_) => {} - Err(err) => { - error!("Error sending to pubsub: {:?}", err.message()) // TODO: retry logic - } - }); - } -} diff --git a/observability/indexer/src/commands/index_accounts.rs b/observability/indexer/src/commands/index_accounts.rs deleted file mode 100644 index c745dea9..00000000 --- a/observability/indexer/src/commands/index_accounts.rs +++ /dev/null @@ -1,413 +0,0 @@ -use crate::utils::{big_query::DATE_FORMAT_STR, convert_account, protos::gcp_pubsub}; -use anyhow::Result; -use base64::{engine::general_purpose, Engine}; -use chrono::{DateTime, Utc}; -use envconfig::Envconfig; -use futures::{future::join_all, StreamExt}; -use google_cloud_default::WithAuthExt; -use google_cloud_googleapis::pubsub::v1::PubsubMessage; -use google_cloud_pubsub::client::{Client, ClientConfig}; -use itertools::Itertools; -use solana_measure::measure::Measure; -use solana_sdk::{account::Account, pubkey::Pubkey, signature::Signature}; -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, Mutex, - }, - time::Duration, -}; -use tonic::Status; -use tracing::{debug, error, info, warn}; -use uuid::Uuid; -use yellowstone_grpc_client::GeyserGrpcClient; -use yellowstone_grpc_proto::geyser::{ - subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequest, - SubscribeRequestFilterAccounts, SubscribeRequestFilterSlots, -}; - -#[derive(Envconfig, Debug, Clone)] -pub struct IndexAccountsConfig { - #[envconfig(from = "INDEX_ACCOUNTS_RPC_ENDPOINT")] - pub rpc_endpoint: String, - #[envconfig(from = "INDEX_ACCOUNTS_RPC_TOKEN")] - pub rpc_token: String, - #[envconfig(from = "INDEX_ACCOUNTS_SLOTS_BUFFER_SIZE")] - pub slots_buffer_size: u32, - #[envconfig(from = "INDEX_ACCOUNTS_MAX_CONCURRENT_REQUESTS")] - pub max_concurrent_requests: usize, - #[envconfig(from = "INDEX_ACCOUNTS_MONITOR_INTERVAL")] - pub monitor_interval: u64, - #[envconfig(from = "INDEX_ACCOUNTS_PROGRAM_ID")] - pub program_id: Pubkey, - - #[envconfig(from = "INDEX_ACCOUNTS_PROJECT_ID")] - pub project_id: String, - #[envconfig(from = "INDEX_ACCOUNTS_PUBSUB_TOPIC_NAME")] - pub topic_name: String, - #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] - pub gcp_sa_key: Option, -} - -#[derive(Debug, Clone)] -pub struct AccountUpdateData { - pub timestamp: DateTime, - pub slot: u64, - pub address: Pubkey, - pub txn_signature: Option, - pub write_version: Option, - pub account_data: Account, -} - -#[derive(Clone)] -pub struct Context { - pub config: Arc, - account_updates_queue: Arc>>>, - account_updates_counter: Arc, - latest_slots_with_commitment: Arc>>, - stream_disconnection_count: Arc, - update_processing_error_count: Arc, -} - -impl Context { - pub async fn new(config: &IndexAccountsConfig) -> Self { - Self { - config: Arc::new(config.clone()), - account_updates_queue: Arc::new(Mutex::new(BTreeMap::new())), - account_updates_counter: Arc::new(AtomicU64::new(0)), - latest_slots_with_commitment: Arc::new(Mutex::new(BTreeSet::new())), - stream_disconnection_count: Arc::new(AtomicU64::new(0)), - update_processing_error_count: Arc::new(AtomicU64::new(0)), - } - } -} - -pub async fn index_accounts(config: IndexAccountsConfig) -> Result<()> { - let context = Arc::new(Context::new(&config).await); - - let listen_to_updates_handle = tokio::spawn({ - let context = context.clone(); - async move { listen_to_updates(context).await } - }); - let process_account_updates_handle = tokio::spawn({ - let context = context.clone(); - async move { push_transactions_to_pubsub(context).await.unwrap() } - }); - let monitor_handle = tokio::spawn({ - let context = context.clone(); - async move { monitor(context).await } - }); - - join_all([ - listen_to_updates_handle, - process_account_updates_handle, - monitor_handle, - ]) - .await; - - Ok(()) -} - -async fn listen_to_updates(ctx: Arc) { - loop { - info!("Connecting geyser client"); - let geyser_client_connection_result = GeyserGrpcClient::connect( - ctx.config.rpc_endpoint.to_string(), - Some(ctx.config.rpc_token.to_string()), - None, - ); - - let mut geyser_client = match geyser_client_connection_result { - Ok(geyser_client) => geyser_client, - Err(err) => { - error!("Error connecting to geyser client: {}", err); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; - } - }; - - let subscribe_request = SubscribeRequest { - accounts: HashMap::from_iter([( - ctx.config.program_id.to_string(), - SubscribeRequestFilterAccounts { - owner: vec![ctx.config.program_id.to_string()], - account: vec![], - ..Default::default() - }, - )]), - slots: HashMap::from_iter([( - "slots".to_string(), - SubscribeRequestFilterSlots::default(), - )]), - ..Default::default() - }; - - // Establish streams - let (_, mut stream) = match geyser_client - .subscribe_with_request(Some(subscribe_request)) - .await - { - Ok(value) => value, - Err(e) => { - error!("Error subscribing geyser client {e}"); - continue; - } - }; - - while let Some(received) = stream.next().await { - match received { - Ok(received) => { - if let Some(update) = received.update_oneof { - match process_update(ctx.clone(), update) { - Ok(_) => {} - Err(err) => { - error!("Error processing update: {}", err); - ctx.update_processing_error_count - .fetch_add(1, Ordering::Relaxed); - } - } - } - } - Err(err) => { - error!("Error pulling next update: {}", err); - tokio::time::sleep(Duration::from_secs(1)).await; - break; - } - } - } - - error!("Stream got disconnected"); - ctx.stream_disconnection_count - .fetch_add(1, Ordering::Relaxed); - } -} - -fn process_update(ctx: Arc, update: UpdateOneof) -> Result<()> { - match update { - UpdateOneof::Account(account_update) => { - let update_slot = account_update.slot; - if let Some(account_info) = account_update.account { - let address = &Pubkey::try_from(account_info.pubkey.clone()).unwrap(); - let txn_signature = account_info - .txn_signature - .clone() - .map(|sig_bytes| Signature::try_from(sig_bytes).unwrap()); - let mut account_updates_queue = ctx.account_updates_queue.lock().unwrap(); - - let slot_account_updates = match account_updates_queue.get_mut(&update_slot) { - Some(slot_account_updates) => slot_account_updates, - None => { - account_updates_queue.insert(update_slot, HashMap::default()); - account_updates_queue.get_mut(&update_slot).unwrap() - } - }; - - slot_account_updates.insert( - *address, - AccountUpdateData { - address: *address, - timestamp: Utc::now(), - slot: update_slot, - txn_signature, - write_version: Some(account_info.write_version), - account_data: convert_account(account_info).unwrap(), - }, - ); - } else { - anyhow::bail!("Expected `transaction` in `UpdateOneof::Transaction` update"); - } - } - UpdateOneof::Slot(slot) => { - if slot.status == CommitmentLevel::Confirmed as i32 - || slot.status == CommitmentLevel::Finalized as i32 - { - let mut latest_slots = ctx.latest_slots_with_commitment.lock().unwrap(); - let slot_inserted = latest_slots.insert(slot.slot); - if slot_inserted && latest_slots.len() > ctx.config.slots_buffer_size as usize { - let oldest_slot = *latest_slots.first().unwrap(); - latest_slots.remove(&oldest_slot); - } - } - } - UpdateOneof::Ping(_) => { - debug!("ping"); - } - _ => { - warn!("unknown update"); - } - } - - Ok(()) -} - -pub async fn push_transactions_to_pubsub(ctx: Arc) -> Result<()> { - let topic_name = ctx.config.topic_name.as_str(); - - let client_config = ClientConfig::default().with_auth().await?; - let client = Client::new(client_config).await?; - - let topic = client.topic(topic_name); - topic - .exists(None, None) - .await - .unwrap_or_else(|_| panic!("topic {} not found", topic_name)); - - let publisher = topic.new_publisher(None); - - loop { - let mut account_updates_data: Vec = vec![]; - { - let mut account_updates_per_slot = ctx.account_updates_queue.lock().unwrap(); - let latest_slots_with_commitment = ctx.latest_slots_with_commitment.lock().unwrap(); - - // Remove all transactions received in a slot that has not been confirmed in allotted time - if let Some(oldest_slot_with_commitment) = latest_slots_with_commitment.first() { - account_updates_per_slot.retain(|slot, account_updates| { - if slot < oldest_slot_with_commitment { - debug!( - "throwing away txs {:?} from slot {}", - account_updates - .iter() - .map(|(address, _)| address.to_string()) - .collect_vec(), - slot - ); - } - - slot >= oldest_slot_with_commitment - }); - } - - // Add transactions from confirmed slots to the queue of transactions to be indexed - for (slot, slot_account_updates) in account_updates_per_slot.clone().iter() { - if let Some(latest_slot_with_commitment) = latest_slots_with_commitment.last() { - if slot > latest_slot_with_commitment { - break; // Ok because transactions_per_slot is sorted (BtreeMap) - } - } - - if latest_slots_with_commitment.contains(slot) { - account_updates_data.extend(slot_account_updates.values().cloned()); - account_updates_per_slot.remove(slot); - } - } - } - - if account_updates_data.is_empty() { - tokio::time::sleep(Duration::from_millis(10)).await; - continue; - } - - let mut messages = vec![]; - - account_updates_data.iter().for_each(|account_update_data| { - ctx.account_updates_counter.fetch_add(1, Ordering::Relaxed); - - let now = Utc::now(); - - let message = gcp_pubsub::PubsubAccountUpdate { - id: Uuid::new_v4().to_string(), - created_at: now.format(DATE_FORMAT_STR).to_string(), - timestamp: account_update_data - .timestamp - .format(DATE_FORMAT_STR) - .to_string(), - owner: account_update_data.account_data.owner.to_string(), - slot: account_update_data.slot, - pubkey: account_update_data.address.to_string(), - txn_signature: account_update_data.txn_signature.map(|sig| sig.to_string()), - write_version: account_update_data.write_version, - lamports: account_update_data.account_data.lamports, - executable: account_update_data.account_data.executable, - rent_epoch: account_update_data.account_data.rent_epoch, - data: general_purpose::STANDARD.encode(&account_update_data.account_data.data), - }; - - let message_str = serde_json::to_string(&message).unwrap(); - let message_bytes = message_str.as_bytes().to_vec(); - messages.push(PubsubMessage { - data: message_bytes.into(), - ..PubsubMessage::default() - }); - }); - - // Send a message. There are also `publish_bulk` and `publish_immediately` methods. - let awaiters = publisher.publish_bulk(messages).await; - - // The get method blocks until a server-generated ID or an error is returned for the published message. - let pub_results: Vec> = join_all( - awaiters - .into_iter() - .map(|awaiter| awaiter.get(None)) - .collect_vec(), - ) - .await; - - pub_results.into_iter().for_each(|result| match result { - Ok(_) => {} - Err(status) => { - error!( - "Error sending tx to pubsub (code {:?}): {:?}", - status.code(), - status.message() - ) - } - }); - } -} - -async fn monitor(ctx: Arc) { - let mut main_timing = Measure::start("main"); - let mut last_fetch_count = 0u64; - let mut last_fetch_time = 0f32; - - loop { - tokio::time::sleep(Duration::from_secs(ctx.config.monitor_interval)).await; - main_timing.stop(); - let latest_slots = ctx.latest_slots_with_commitment.lock().unwrap().clone(); - let account_updates_queue = ctx.account_updates_queue.lock().unwrap().clone(); - let earliest_block_with_commitment = latest_slots.first().unwrap_or(&0); - let latest_block_with_commitment = latest_slots.last().unwrap_or(&u64::MAX); - let earliest_pending_slot = account_updates_queue - .first_key_value() - .map(|(slot, _)| slot) - .unwrap_or(&0); - let latest_pending_slot = account_updates_queue - .first_key_value() - .map(|(slot, _)| slot) - .unwrap_or(&u64::MAX); - let current_fetch_count = ctx.account_updates_counter.load(Ordering::Relaxed); - let stream_disconnection_count = ctx.stream_disconnection_count.load(Ordering::Relaxed); - let update_processing_error_count = - ctx.update_processing_error_count.load(Ordering::Relaxed); - let current_fetch_time = main_timing.as_s(); - - let ingest_rate = if (current_fetch_time - last_fetch_time) > 0.0 { - (current_fetch_count - last_fetch_count) as f32 / (current_fetch_time - last_fetch_time) - } else { - f32::INFINITY - }; - let account_updates_queue_size = ctx.account_updates_queue.lock().unwrap().len(); - - debug!( - "Time: {:.1}s | Total account udpates: {} | {:.1}s count: {} | {:.1}s rate: {:.1} tx/s | Tx Q size: {} | Stream disconnections: {} | Processing errors: {} | Earliest confirmed slot: {} | Latest confirmed slot: {} | Earliest pending slot: {} | Latest pending slot: {}", - current_fetch_time, - current_fetch_count, - current_fetch_time - last_fetch_time, - current_fetch_count - last_fetch_count, - current_fetch_time - last_fetch_time, - ingest_rate, - account_updates_queue_size, - stream_disconnection_count, - update_processing_error_count, - earliest_block_with_commitment, - latest_block_with_commitment, - earliest_pending_slot, - latest_pending_slot, - ); - - last_fetch_count = current_fetch_count; - last_fetch_time = current_fetch_time; - } -} diff --git a/observability/indexer/src/commands/index_transactions.rs b/observability/indexer/src/commands/index_transactions.rs deleted file mode 100644 index a5872bf9..00000000 --- a/observability/indexer/src/commands/index_transactions.rs +++ /dev/null @@ -1,453 +0,0 @@ -use crate::utils::{big_query::DATE_FORMAT_STR, protos::gcp_pubsub}; -use anyhow::Result; -use base64::{engine::general_purpose, Engine}; -use chrono::{DateTime, Utc}; -use envconfig::Envconfig; -use futures::{future::join_all, StreamExt}; -use google_cloud_default::WithAuthExt; -use google_cloud_googleapis::pubsub::v1::PubsubMessage; -use google_cloud_pubsub::client::{Client, ClientConfig}; -use itertools::Itertools; -use solana_measure::measure::Measure; -use solana_sdk::{pubkey::Pubkey, signature::Signature, transaction::TransactionVersion}; -use solana_transaction_status::{ - TransactionWithStatusMeta, UiTransactionStatusMeta, VersionedTransactionWithStatusMeta, -}; -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, Mutex, - }, - time::Duration, -}; -use tonic::Status; -use tracing::{debug, error, info, warn}; -use uuid::Uuid; -use yellowstone_grpc_client::GeyserGrpcClient; -use yellowstone_grpc_proto::{ - convert_from, - geyser::{ - subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequest, - SubscribeRequestFilterSlots, SubscribeRequestFilterTransactions, - }, -}; - -#[derive(Envconfig, Debug, Clone)] -pub struct IndexTransactionsConfig { - #[envconfig(from = "INDEX_TRANSACTIONS_RPC_ENDPOINT")] - pub rpc_endpoint: String, - #[envconfig(from = "INDEX_TRANSACTIONS_RPC_TOKEN")] - pub rpc_token: String, - #[envconfig(from = "INDEX_TRANSACTIONS_SLOTS_BUFFER_SIZE")] - pub slots_buffer_size: u32, - #[envconfig(from = "INDEX_TRANSACTIONS_MONITOR_INTERVAL")] - pub monitor_interval: u64, - #[envconfig(from = "INDEX_TRANSACTIONS_PROGRAM_ID")] - pub program_id: Pubkey, - - #[envconfig(from = "INDEX_TRANSACTIONS_PROJECT_ID")] - pub project_id: String, - #[envconfig(from = "INDEX_TRANSACTIONS_PUBSUB_TOPIC_NAME")] - pub topic_name: String, - #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] - pub gcp_sa_key: Option, -} - -#[derive(Debug, Clone)] -pub struct TransactionData { - pub timestamp: DateTime, - pub slot: u64, - pub signature: Signature, - pub indexing_addresses: Vec, - pub transaction: VersionedTransactionWithStatusMeta, -} - -#[derive(Clone)] -pub struct Context { - pub config: Arc, - transactions_queue: Arc>>>, - transactions_counter: Arc, - latest_slots_with_commitment: Arc>>, - stream_disconnection_count: Arc, - update_processing_error_count: Arc, -} - -impl Context { - pub async fn new(config: &IndexTransactionsConfig) -> Self { - Self { - config: Arc::new(config.clone()), - transactions_queue: Arc::new(Mutex::new(BTreeMap::new())), - transactions_counter: Arc::new(AtomicU64::new(0)), - latest_slots_with_commitment: Arc::new(Mutex::new(BTreeSet::new())), - stream_disconnection_count: Arc::new(AtomicU64::new(0)), - update_processing_error_count: Arc::new(AtomicU64::new(0)), - } - } -} - -pub async fn index_transactions(config: IndexTransactionsConfig) -> Result<()> { - let context = Arc::new(Context::new(&config).await); - - let listen_to_updates_handle = tokio::spawn({ - let context = context.clone(); - async move { listen_to_updates(context).await } - }); - let process_transactions_handle = tokio::spawn({ - let context = context.clone(); - async move { push_transactions_to_pubsub(context).await.unwrap() } - }); - let monitor_handle = tokio::spawn({ - let context = context.clone(); - async move { monitor(context).await } - }); - - join_all([ - listen_to_updates_handle, - process_transactions_handle, - monitor_handle, - ]) - .await; - - Ok(()) -} - -async fn listen_to_updates(ctx: Arc) { - loop { - info!("Connecting geyser client"); - let geyser_client_connection_result = GeyserGrpcClient::connect_with_timeout( - ctx.config.rpc_endpoint.to_string(), - Some(ctx.config.rpc_token.to_string()), - None, - Some(Duration::from_secs(10)), - Some(Duration::from_secs(10)), - false, - ) - .await; - info!("Connected"); - - let mut geyser_client = match geyser_client_connection_result { - Ok(geyser_client) => geyser_client, - Err(err) => { - error!("Error connecting to geyser client: {}", err); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; - } - }; - - let subscribe_request = SubscribeRequest { - slots: HashMap::from_iter([( - "client".to_string(), - SubscribeRequestFilterSlots { - filter_by_commitment: Some(false), - }, - )]), - transactions: HashMap::from_iter([( - ctx.config.program_id.to_string(), - SubscribeRequestFilterTransactions { - vote: Some(false), - failed: Some(false), - account_include: vec![ctx.config.program_id.to_string()], - account_exclude: vec![], - ..Default::default() - }, - )]), - commitment: Some(CommitmentLevel::Processed as i32), - ..Default::default() - }; - - // Establish streams - let (_, mut stream) = match geyser_client - .subscribe_with_request(Some(subscribe_request)) - .await - { - Ok(value) => value, - Err(e) => { - error!("Error subscribing geyser client {e}"); - continue; - } - }; - - while let Some(received) = stream.next().await { - match received { - Ok(received) => { - if let Some(update) = received.update_oneof { - match process_update(ctx.clone(), &received.filters, update) { - Ok(_) => {} - Err(err) => { - error!("Error processing update: {}", err); - ctx.update_processing_error_count - .fetch_add(1, Ordering::Relaxed); - } - } - } - } - Err(err) => { - error!("Error pulling next update: {}", err); - tokio::time::sleep(Duration::from_secs(1)).await; - break; - } - } - } - - error!("Stream got disconnected"); - ctx.stream_disconnection_count - .fetch_add(1, Ordering::Relaxed); - } -} - -fn process_update(ctx: Arc, filters: &[String], update: UpdateOneof) -> Result<()> { - match update { - UpdateOneof::Transaction(transaction_update) => { - if let Some(transaction_info) = transaction_update.transaction { - let signature = transaction_info.signature.clone(); - let transaction = convert_from::create_tx_with_meta(transaction_info).unwrap(); - let mut transactions_queue = ctx.transactions_queue.lock().unwrap(); - - let slot_transactions = match transactions_queue.get_mut(&transaction_update.slot) { - Some(slot_transactions) => slot_transactions, - None => { - transactions_queue.insert(transaction_update.slot, vec![]); - transactions_queue - .get_mut(&transaction_update.slot) - .unwrap() - } - }; - - let transaction = match transaction { - TransactionWithStatusMeta::MissingMetadata(transaction) => { - error!( - "Missing metadata for transaction {}. Skipping potentially relevant transaction.", - transaction.signatures.first().unwrap() - ); - return Ok(()); - } - TransactionWithStatusMeta::Complete(transaction_with_meta) => { - transaction_with_meta - } - }; - - slot_transactions.push(TransactionData { - timestamp: Utc::now(), - signature: Signature::try_from(signature).unwrap(), - slot: transaction_update.slot, - indexing_addresses: filters.to_vec(), - transaction, - }); - } else { - anyhow::bail!("Expected `transaction` in `UpdateOneof::Transaction` update"); - } - } - UpdateOneof::Slot(slot) => { - if slot.status == CommitmentLevel::Confirmed as i32 - || slot.status == CommitmentLevel::Finalized as i32 - { - let mut latest_slots = ctx.latest_slots_with_commitment.lock().unwrap(); - let slot_inserted = latest_slots.insert(slot.slot); - if slot_inserted && latest_slots.len() > ctx.config.slots_buffer_size as usize { - let oldest_slot = *latest_slots.first().unwrap(); - latest_slots.remove(&oldest_slot); - } - } - } - UpdateOneof::Ping(_) => { - debug!("ping"); - } - _ => { - warn!("unknown update: {:?}", update); - } - } - - Ok(()) -} - -pub async fn push_transactions_to_pubsub(ctx: Arc) -> Result<()> { - let topic_name = ctx.config.topic_name.as_str(); - - let client_config = ClientConfig::default().with_auth().await?; - let client = Client::new(client_config).await.unwrap(); - - let topic = client.topic(topic_name); - topic - .exists(None, None) - .await - .unwrap_or_else(|_| panic!("topic {} not found", topic_name)); - - let publisher = topic.new_publisher(None); - - loop { - let mut transactions_data: Vec = vec![]; - { - let mut transactions_per_slot = ctx.transactions_queue.lock().unwrap(); - let latest_slots_with_commitment = ctx.latest_slots_with_commitment.lock().unwrap(); - - // Remove all transactions received in a slot that has not been confirmed in allotted time - if let Some(oldest_slot_with_commitment) = latest_slots_with_commitment.first() { - transactions_per_slot.retain(|slot, transactions| { - if slot < oldest_slot_with_commitment { - debug!( - "throwing away txs {:?} from slot {}", - transactions - .iter() - .map(|tx| tx.signature.to_string()) - .collect_vec(), - slot - ); - } - - slot >= oldest_slot_with_commitment - }); - } - - // Add transactions from confirmed slots to the queue of transactions to be indexed - for (slot, slot_transactions) in transactions_per_slot.clone().iter() { - if let Some(latest_slot_with_commitment) = latest_slots_with_commitment.last() { - if slot > latest_slot_with_commitment { - break; // Ok because transactions_per_slot is sorted (BtreeMap) - } - } - - if latest_slots_with_commitment.contains(slot) { - transactions_data.extend(slot_transactions.clone()); - transactions_per_slot.remove(slot); - } - } - } - - if transactions_data.is_empty() { - tokio::time::sleep(Duration::from_millis(10)).await; - continue; - } - - let mut messages = vec![]; - - transactions_data.iter().for_each(|transaction_data| { - ctx.transactions_counter.fetch_add(1, Ordering::Relaxed); - - let now = Utc::now(); - - transaction_data - .indexing_addresses - .iter() - .for_each(|indexing_address| { - let message = gcp_pubsub::PubsubTransaction { - id: Uuid::new_v4().to_string(), - created_at: now.format(DATE_FORMAT_STR).to_string(), - timestamp: transaction_data - .timestamp - .format(DATE_FORMAT_STR) - .to_string(), - signature: transaction_data.signature.to_string(), - indexing_address: indexing_address.to_string(), - slot: transaction_data.slot, - signer: transaction_data - .transaction - .transaction - .message - .static_account_keys() - .first() - .unwrap() - .to_string(), - success: transaction_data.transaction.meta.status.is_ok(), - version: match transaction_data.transaction.transaction.version() { - TransactionVersion::Legacy(_) => "legacy".to_string(), - TransactionVersion::Number(version) => version.to_string(), - }, - - fee: transaction_data.transaction.meta.fee, - meta: serde_json::to_string(&UiTransactionStatusMeta::from( - transaction_data.transaction.meta.clone(), - )) - .unwrap(), - message: general_purpose::STANDARD - .encode(transaction_data.transaction.transaction.message.serialize()), - }; - - let message_str = serde_json::to_string(&message).unwrap(); - let message_bytes = message_str.as_bytes().to_vec(); - messages.push(PubsubMessage { - data: message_bytes.into(), - ..PubsubMessage::default() - }); - }); - }); - - // Send a message. There are also `publish_bulk` and `publish_immediately` methods. - let awaiters = publisher.publish_bulk(messages).await; - - // The get method blocks until a server-generated ID or an error is returned for the published message. - let pub_results: Vec> = join_all( - awaiters - .into_iter() - .map(|awaiter| awaiter.get(None)) - .collect_vec(), - ) - .await; - - pub_results.into_iter().for_each(|result| match result { - Ok(_) => {} - Err(status) => { - error!( - "Error sending tx to pubsub (code {:?}): {:?}", - status.code(), - status.message() - ) - } - }); - } -} - -async fn monitor(ctx: Arc) { - let mut main_timing = Measure::start("main"); - let mut last_fetch_count = 0u64; - let mut last_fetch_time = 0f32; - - loop { - tokio::time::sleep(Duration::from_secs(ctx.config.monitor_interval)).await; - main_timing.stop(); - let latest_slots = ctx.latest_slots_with_commitment.lock().unwrap().clone(); - let tx_queue = ctx.transactions_queue.lock().unwrap().clone(); - let earliest_block_with_commitment = latest_slots.first().unwrap_or(&0); - let latest_block_with_commitment = latest_slots.last().unwrap_or(&u64::MAX); - let earliest_pending_slot = tx_queue - .first_key_value() - .map(|(slot, _)| slot) - .unwrap_or(&0); - let latest_pending_slot = tx_queue - .first_key_value() - .map(|(slot, _)| slot) - .unwrap_or(&u64::MAX); - let current_fetch_count = ctx.transactions_counter.load(Ordering::Relaxed); - let stream_disconnection_count = ctx.stream_disconnection_count.load(Ordering::Relaxed); - let update_processing_error_count = - ctx.update_processing_error_count.load(Ordering::Relaxed); - let current_fetch_time = main_timing.as_s(); - - let ingest_rate = if (current_fetch_time - last_fetch_time) > 0.0 { - (current_fetch_count - last_fetch_count) as f32 / (current_fetch_time - last_fetch_time) - } else { - f32::INFINITY - }; - let tx_queue_size = ctx.transactions_queue.lock().unwrap().len(); - - debug!( - "Time: {:.1}s | Total txs: {} | {:.1}s count: {} | {:.1}s rate: {:.1} tx/s | Tx Q size: {} | Stream disconnections: {} | Processing errors: {} | Earliest confirmed slot: {} | Latest confirmed slot: {} | Earliest pending slot: {} | Latest pending slot: {}", - current_fetch_time, - current_fetch_count, - current_fetch_time - last_fetch_time, - current_fetch_count - last_fetch_count, - current_fetch_time - last_fetch_time, - ingest_rate, - tx_queue_size, - stream_disconnection_count, - update_processing_error_count, - earliest_block_with_commitment, - latest_block_with_commitment, - earliest_pending_slot, - latest_pending_slot, - ); - - last_fetch_count = current_fetch_count; - last_fetch_time = current_fetch_time; - } -} diff --git a/observability/indexer/src/utils/protos.rs b/observability/indexer/src/utils/protos.rs deleted file mode 100644 index ad48e4c9..00000000 --- a/observability/indexer/src/utils/protos.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod gcp_pubsub { - tonic::include_proto!("gcp_pubsub"); -} From 7dcc3abed36d26dc47fa7601ff18710f8caec80c Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:44:59 +0800 Subject: [PATCH 02/20] feat: wip db storage --- observability/Cargo.lock | 94 +++++++++- observability/Cargo.toml | 8 + observability/crates/event_indexer/Cargo.toml | 7 +- .../crates/event_indexer/bin/index_events.rs | 15 +- .../crates/event_indexer/diesel.toml | 9 + .../crates/event_indexer/migrations/.keep | 0 .../down.sql | 6 + .../up.sql | 36 ++++ .../2024-03-14-151141_initial_tables/down.sql | 13 ++ .../2024-03-14-151141_initial_tables/up.sql | 109 ++++++++++++ .../crates/event_indexer/src/db/mod.rs | 10 ++ .../crates/event_indexer/src/db/models.rs | 135 ++++++++++++++ .../crates/event_indexer/src/db/schema.rs | 166 ++++++++++++++++++ .../crates/event_indexer/src/indexer.rs | 82 ++++++++- observability/crates/event_indexer/src/lib.rs | 3 +- .../crates/event_indexer/src/parser.rs | 27 ++- 16 files changed, 688 insertions(+), 32 deletions(-) create mode 100644 observability/crates/event_indexer/diesel.toml create mode 100644 observability/crates/event_indexer/migrations/.keep create mode 100644 observability/crates/event_indexer/migrations/00000000000000_diesel_initial_setup/down.sql create mode 100644 observability/crates/event_indexer/migrations/00000000000000_diesel_initial_setup/up.sql create mode 100644 observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/down.sql create mode 100644 observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql create mode 100644 observability/crates/event_indexer/src/db/mod.rs create mode 100644 observability/crates/event_indexer/src/db/models.rs create mode 100644 observability/crates/event_indexer/src/db/schema.rs diff --git a/observability/Cargo.lock b/observability/Cargo.lock index 29ef9fd4..cea5538e 100644 --- a/observability/Cargo.lock +++ b/observability/Cargo.lock @@ -681,6 +681,19 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bigdecimal" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9324c8014cd04590682b34f1e9448d38f0674d0f7b2dc553331016ef0e4e9ebc" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.4", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -1322,6 +1335,43 @@ dependencies = [ "zeroize", ] +[[package]] +name = "diesel" +version = "2.1.4" +source = "git+https://github.com/diesel-rs/diesel.git?rev=12429cb4eebfb50581c494e1a183e4faed65c47f#12429cb4eebfb50581c494e1a183e4faed65c47f" +dependencies = [ + "bigdecimal", + "bitflags 2.4.2", + "byteorder", + "chrono", + "diesel_derives", + "itoa", + "num-bigint 0.4.4", + "num-integer", + "num-traits", + "pq-sys", +] + +[[package]] +name = "diesel_derives" +version = "2.1.0" +source = "git+https://github.com/diesel-rs/diesel.git?rev=12429cb4eebfb50581c494e1a183e4faed65c47f#12429cb4eebfb50581c494e1a183e4faed65c47f" +dependencies = [ + "diesel_table_macro_syntax", + "dsl_auto_type", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + +[[package]] +name = "diesel_table_macro_syntax" +version = "0.1.0" +source = "git+https://github.com/diesel-rs/diesel.git?rev=12429cb4eebfb50581c494e1a183e4faed65c47f#12429cb4eebfb50581c494e1a183e4faed65c47f" +dependencies = [ + "syn 2.0.52", +] + [[package]] name = "digest" version = "0.9.0" @@ -1382,6 +1432,19 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dsl_auto_type" +version = "0.1.0" +source = "git+https://github.com/diesel-rs/diesel.git?rev=12429cb4eebfb50581c494e1a183e4faed65c47f#12429cb4eebfb50581c494e1a183e4faed65c47f" +dependencies = [ + "darling", + "either", + "heck 0.4.1", + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.52", +] + [[package]] name = "dyn-clone" version = "1.0.17" @@ -1544,12 +1607,17 @@ dependencies = [ "anchor-lang", "backoff", "bytemuck", + "chrono", "crossbeam", + "diesel", "dotenv", "envconfig", "futures", "marginfi", "rpc_utils", + "rust_decimal", + "serde", + "serde_json", "solana-account-decoder", "solana-client", "solana-measure", @@ -2233,6 +2301,12 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libsecp256k1" version = "0.6.0" @@ -2871,6 +2945,15 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pq-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb954aedd3cf61784ee8e2ee2e8fbd74c445efc3a7405967d1834864d7bf873a" +dependencies = [ + "vcpkg", +] + [[package]] name = "prettyplease" version = "0.2.16" @@ -3398,6 +3481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee9164faf726e4f3ece4978b25ca877ddc6802fa77f38cdccb32c7f805ecd70c" dependencies = [ "arrayvec", + "diesel", "num-traits", "serde", ] @@ -5168,9 +5252,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -5586,6 +5670,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vec_map" version = "0.8.2" diff --git a/observability/Cargo.toml b/observability/Cargo.toml index 2b501722..6a77238f 100644 --- a/observability/Cargo.toml +++ b/observability/Cargo.toml @@ -6,8 +6,10 @@ members = ["crates/*"] anchor-lang = "0.28.0" backoff = { version = "0.4.0", features = ["tokio"] } bytemuck = "1.13.1" +chrono = "0.4.31" concurrent-queue = "2.0.0" crossbeam = "0.8.2" +diesel = { git = "https://github.com/diesel-rs/diesel.git", rev = "12429cb4eebfb50581c494e1a183e4faed65c47f" } dotenv = "0.15.0" futures = "0.3.25" envconfig = "0.10.0" @@ -15,6 +17,9 @@ marginfi = { path = "../programs/marginfi", features = [ "no-entrypoint", "client", ] } +rust_decimal = { version = "1.26.1", features = ["db-diesel2-postgres"] } +serde = "1.0.197" +serde_json = "1.0.114" solana-account-decoder = "1.16.23" solana-client = "1.16.23" solana-measure = "1.16.23" @@ -30,3 +35,6 @@ tracing-stackdriver = "0.6.1" tracing-subscriber = { version = "0.3.15", features = ["env-filter", "fmt"] } yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", rev = "a2cd1498ac64baa1017d4a4cdefbf46100215b4c" } yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", rev = "a2cd1498ac64baa1017d4a4cdefbf46100215b4c" } + +[patch.crates-io] +diesel = { git = "https://github.com/diesel-rs/diesel.git", rev = "12429cb4eebfb50581c494e1a183e4faed65c47f" } diff --git a/observability/crates/event_indexer/Cargo.toml b/observability/crates/event_indexer/Cargo.toml index b0af5704..2958757a 100644 --- a/observability/crates/event_indexer/Cargo.toml +++ b/observability/crates/event_indexer/Cargo.toml @@ -19,12 +19,17 @@ mainnet-beta = ["marginfi/mainnet-beta"] anchor-lang = { workspace = true } backoff = { workspace = true } bytemuck = { workspace = true } +chrono = { workspace = true } crossbeam = { workspace = true } +diesel = { workspace = true, features = ["postgres", "chrono", "numeric"] } dotenv = { workspace = true } -futures = { workspace = true } envconfig = { workspace = true } +futures = { workspace = true } marginfi = { workspace = true } rpc_utils = { path = "../rpc_utils" } +rust_decimal = { workspace = true, features = ["db-diesel2-postgres"] } +serde = { workspace = true } +serde_json = { workspace = true } solana-account-decoder = { workspace = true } solana-client = { workspace = true } solana-measure = { workspace = true } diff --git a/observability/crates/event_indexer/bin/index_events.rs b/observability/crates/event_indexer/bin/index_events.rs index 55bce8a0..77560cf4 100644 --- a/observability/crates/event_indexer/bin/index_events.rs +++ b/observability/crates/event_indexer/bin/index_events.rs @@ -1,6 +1,6 @@ use dotenv::dotenv; use envconfig::Envconfig; -use event_indexer::{error::IndexingError, indexer::EventIndexer, snapshot::Snapshot}; +use event_indexer::{error::IndexingError, indexer::EventIndexer}; use tracing::{error, info}; use tracing_subscriber::{layer::SubscriberExt, EnvFilter}; @@ -16,6 +16,8 @@ pub struct Config { pub monitor_interval: u64, #[envconfig(from = "PRETTY_LOGS")] pub pretty_logs: Option, + #[envconfig(from = "DATABASE_URL")] + pub database_url: String, // #[envconfig(from = "INDEX_TRANSACTIONS_PROJECT_ID")] // pub project_id: String, // #[envconfig(from = "INDEX_TRANSACTIONS_PUBSUB_TOPIC_NAME")] @@ -52,16 +54,15 @@ pub async fn main() -> Result<(), IndexingError> { let mut indexer = EventIndexer::new( config.rpc_host.clone(), config.rpc_token.clone(), - "".to_string(), + config.database_url, ); let first_sig = indexer.init().await; info!("First signature: {:?}", first_sig); - let rpc_endpoint = format!("{}/{}", config.rpc_host, config.rpc_token).to_string(); - let mut snapshot = Snapshot::new(marginfi::ID, rpc_endpoint); - snapshot.init().await?; - - println!("Snapshot: {}", snapshot); + // let rpc_endpoint = format!("{}/{}", config.rpc_host, config.rpc_token).to_string(); + // let mut snapshot = Snapshot::new(marginfi::ID, rpc_endpoint); + // snapshot.init().await?; + // println!("Snapshot: {}", snapshot); let indexer_handle = tokio::spawn(async move { indexer.run().await }); diff --git a/observability/crates/event_indexer/diesel.toml b/observability/crates/event_indexer/diesel.toml new file mode 100644 index 00000000..fa8cc692 --- /dev/null +++ b/observability/crates/event_indexer/diesel.toml @@ -0,0 +1,9 @@ +# For documentation on how to configure this file, +# see https://diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/db/schema.rs" +custom_type_derives = ["diesel::query_builder::QueryId"] + +[migrations_directory] +dir = "migrations" diff --git a/observability/crates/event_indexer/migrations/.keep b/observability/crates/event_indexer/migrations/.keep new file mode 100644 index 00000000..e69de29b diff --git a/observability/crates/event_indexer/migrations/00000000000000_diesel_initial_setup/down.sql b/observability/crates/event_indexer/migrations/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 00000000..a9f52609 --- /dev/null +++ b/observability/crates/event_indexer/migrations/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/observability/crates/event_indexer/migrations/00000000000000_diesel_initial_setup/up.sql b/observability/crates/event_indexer/migrations/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 00000000..d68895b1 --- /dev/null +++ b/observability/crates/event_indexer/migrations/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/down.sql b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/down.sql new file mode 100644 index 00000000..afa8962f --- /dev/null +++ b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/down.sql @@ -0,0 +1,13 @@ +-- This file should undo anything in `up.sql` + +DROP TABLE IF EXISTS "withdraw_emissions_events"; +DROP TABLE IF EXISTS "withdraw_events"; +DROP TABLE IF EXISTS "repay_events"; +DROP TABLE IF EXISTS "borrow_events"; +DROP TABLE IF EXISTS "deposit_events"; +DROP TABLE IF EXISTS "transfer_account_authority_events"; +DROP TABLE IF EXISTS "create_account_events"; +DROP TABLE IF EXISTS "accounts"; +DROP TABLE IF EXISTS "users"; +DROP TABLE IF EXISTS "banks"; +DROP TABLE IF EXISTS "mints"; diff --git a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql new file mode 100644 index 00000000..e974eefb --- /dev/null +++ b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql @@ -0,0 +1,109 @@ +-- Your SQL goes here + +CREATE TABLE "mints"( + "id" SERIAL PRIMARY KEY, + "address" VARCHAR NOT NULL, + "symbol" VARCHAR NOT NULL, + "decimals" SMALLINT NOT NULL +); + +CREATE TABLE "banks"( + "id" SERIAL PRIMARY KEY, + "address" VARCHAR NOT NULL UNIQUE, + "mint" INT4 NOT NULL REFERENCES "mints"("id") +); + +CREATE TABLE "users"( + "id" SERIAL PRIMARY KEY, + "address" VARCHAR NOT NULL UNIQUE +); + +CREATE TABLE "accounts"( + "id" SERIAL PRIMARY KEY, + "address" VARCHAR NOT NULL UNIQUE, + "user_id" INT4 NOT NULL REFERENCES "users"("id") +); + +CREATE TABLE "create_account_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), + "authority_id" INT4 NOT NULL REFERENCES "users"("id") +); + +CREATE TABLE "transfer_account_authority_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), + "old_authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "new_authority_id" INT4 NOT NULL REFERENCES "users"("id") +); + +CREATE TABLE "deposit_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), + "authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), + "amount" NUMERIC NOT NULL +); + +CREATE TABLE "borrow_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), + "authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), + "amount" NUMERIC NOT NULL +); + +CREATE TABLE "repay_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), + "authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), + "amount" NUMERIC NOT NULL, + "all" BOOLEAN NOT NULL +); + +CREATE TABLE "withdraw_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), + "authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), + "amount" NUMERIC NOT NULL, + "all" BOOLEAN NOT NULL +); + +CREATE TABLE "withdraw_emissions_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), + "authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), + "emission_mint" VARCHAR NOT NULL, + "amount" NUMERIC NOT NULL +); diff --git a/observability/crates/event_indexer/src/db/mod.rs b/observability/crates/event_indexer/src/db/mod.rs new file mode 100644 index 00000000..cf0b9812 --- /dev/null +++ b/observability/crates/event_indexer/src/db/mod.rs @@ -0,0 +1,10 @@ +use diesel::{Connection, PgConnection}; + +pub mod models; +pub mod schema; + +#[cold] +pub fn establish_connection(database_url: String) -> PgConnection { + PgConnection::establish(&database_url) + .unwrap_or_else(|err| panic!("Error connecting to {}: {:?}", database_url, err)) +} diff --git a/observability/crates/event_indexer/src/db/models.rs b/observability/crates/event_indexer/src/db/models.rs new file mode 100644 index 00000000..89ed7414 --- /dev/null +++ b/observability/crates/event_indexer/src/db/models.rs @@ -0,0 +1,135 @@ +use crate::db::schema::*; +use diesel::prelude::*; +use rust_decimal::Decimal; + +#[derive(Default, Queryable, Insertable)] +#[diesel(table_name = banks)] +pub struct Banks { + #[diesel(skip_insertion)] + pub id: i32, + pub address: String, + pub mint: i32, +} + +#[derive(Default, Queryable, Insertable)] +#[diesel(table_name = users)] +pub struct Users { + #[diesel(skip_insertion)] + pub id: i32, + pub address: String, +} + +#[derive(Default, Queryable, Insertable)] +#[diesel(table_name = accounts)] +pub struct Accounts { + #[diesel(skip_insertion)] + pub id: i32, + pub address: String, + pub user_id: i32, +} + +#[derive(Default, Queryable, Insertable, Associations)] +#[diesel(belongs_to(Accounts, foreign_key = account_id), belongs_to(Users, foreign_key = authority_id))] +#[diesel(table_name = create_account_events)] +pub struct CreateAccountEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + pub account_id: i32, + pub authority_id: i32, +} + +#[derive(Default, Queryable, Insertable)] +#[diesel(table_name = transfer_account_authority_events)] +pub struct TransferAccountAuthorityEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + pub account_id: i32, + pub old_authority_id: i32, + pub new_authority_id: i32, +} + +#[derive(Default, Queryable, Insertable)] +#[diesel(table_name = deposit_events)] +pub struct DepositEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + pub account_id: i32, + pub authority_id: i32, + pub bank_id: i32, + pub amount: Decimal, +} + +#[derive(Default, Queryable, Insertable)] +#[diesel(table_name = borrow_events)] +pub struct BorrowEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + pub account_id: i32, + pub authority_id: i32, + pub bank_id: i32, + pub amount: Decimal, +} + +#[derive(Default, Queryable, Insertable)] +#[diesel(table_name = repay_events)] +pub struct RepayEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + pub account_id: i32, + pub authority_id: i32, + pub bank_id: i32, + pub amount: Decimal, + pub all: bool, +} + +#[derive(Default, Queryable, Insertable)] +#[diesel(table_name = withdraw_events)] +pub struct WithdrawEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + pub account_id: i32, + pub authority_id: i32, + pub bank_id: i32, + pub amount: Decimal, + pub all: bool, +} + +#[derive(Default, Queryable, Insertable)] +#[diesel(table_name = withdraw_emissions_events)] +pub struct WithdrawEmissionsEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + pub account_id: i32, + pub authority_id: i32, + pub bank_id: i32, + pub emission_mint: String, + pub amount: Decimal, +} diff --git a/observability/crates/event_indexer/src/db/schema.rs b/observability/crates/event_indexer/src/db/schema.rs new file mode 100644 index 00000000..94708b47 --- /dev/null +++ b/observability/crates/event_indexer/src/db/schema.rs @@ -0,0 +1,166 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + accounts (id) { + id -> Int4, + address -> Varchar, + user_id -> Int4, + } +} + +diesel::table! { + banks (id) { + id -> Int4, + address -> Varchar, + mint -> Int4, + } +} + +diesel::table! { + borrow_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + account_id -> Int4, + authority_id -> Int4, + bank_id -> Int4, + amount -> Numeric, + } +} + +diesel::table! { + create_account_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + account_id -> Int4, + authority_id -> Int4, + } +} + +diesel::table! { + deposit_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + account_id -> Int4, + authority_id -> Int4, + bank_id -> Int4, + amount -> Numeric, + } +} + +diesel::table! { + mints (id) { + id -> Int4, + address -> Varchar, + symbol -> Varchar, + decimals -> Int2, + } +} + +diesel::table! { + repay_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + account_id -> Int4, + authority_id -> Int4, + bank_id -> Int4, + amount -> Numeric, + all -> Bool, + } +} + +diesel::table! { + transfer_account_authority_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + account_id -> Int4, + old_authority_id -> Int4, + new_authority_id -> Int4, + } +} + +diesel::table! { + users (id) { + id -> Int4, + address -> Varchar, + } +} + +diesel::table! { + withdraw_emissions_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + account_id -> Int4, + authority_id -> Int4, + bank_id -> Int4, + emission_mint -> Varchar, + amount -> Numeric, + } +} + +diesel::table! { + withdraw_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + account_id -> Int4, + authority_id -> Int4, + bank_id -> Int4, + amount -> Numeric, + all -> Bool, + } +} + +diesel::joinable!(accounts -> users (user_id)); +diesel::joinable!(banks -> mints (mint)); +diesel::joinable!(borrow_events -> accounts (account_id)); +diesel::joinable!(borrow_events -> banks (bank_id)); +diesel::joinable!(borrow_events -> users (authority_id)); +diesel::joinable!(create_account_events -> accounts (account_id)); +diesel::joinable!(create_account_events -> users (authority_id)); +diesel::joinable!(deposit_events -> accounts (account_id)); +diesel::joinable!(deposit_events -> banks (bank_id)); +diesel::joinable!(deposit_events -> users (authority_id)); +diesel::joinable!(repay_events -> accounts (account_id)); +diesel::joinable!(repay_events -> banks (bank_id)); +diesel::joinable!(repay_events -> users (authority_id)); +diesel::joinable!(transfer_account_authority_events -> accounts (account_id)); +diesel::joinable!(withdraw_emissions_events -> accounts (account_id)); +diesel::joinable!(withdraw_emissions_events -> banks (bank_id)); +diesel::joinable!(withdraw_emissions_events -> users (authority_id)); +diesel::joinable!(withdraw_events -> accounts (account_id)); +diesel::joinable!(withdraw_events -> banks (bank_id)); +diesel::joinable!(withdraw_events -> users (authority_id)); + +diesel::allow_tables_to_appear_in_same_query!( + accounts, + banks, + borrow_events, + create_account_events, + deposit_events, + mints, + repay_events, + transfer_account_authority_events, + users, + withdraw_emissions_events, + withdraw_events, +); diff --git a/observability/crates/event_indexer/src/indexer.rs b/observability/crates/event_indexer/src/indexer.rs index 32c1a218..e5e7ea82 100644 --- a/observability/crates/event_indexer/src/indexer.rs +++ b/observability/crates/event_indexer/src/indexer.rs @@ -5,12 +5,14 @@ use std::{ vec, }; +use chrono::DateTime; use crossbeam::channel::{Receiver, Sender}; +use diesel::{PgConnection, RunQueryDsl}; use futures::StreamExt; use solana_client::rpc_client::SerializableTransaction; use solana_sdk::{pubkey::Pubkey, signature::Signature}; use solana_transaction_status::{TransactionWithStatusMeta, VersionedTransactionWithStatusMeta}; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; use yellowstone_grpc_client::GeyserGrpcClient; use yellowstone_grpc_proto::{ convert_from, @@ -20,13 +22,21 @@ use yellowstone_grpc_proto::{ }, }; +use crate::{ + db::{ + establish_connection, + models::{Accounts, CreateAccountEvents, Users}, + schema::*, + }, + parser::Event, +}; + use super::parser::{MarginfiEventParser, MarginfiEventWithMeta, MARGINFI_GROUP_ADDRESS}; const BLOCK_META_BUFFER_LENGTH: usize = 30; pub struct EventIndexer { parser: MarginfiEventParser, - // database_connection: PgConnection, transaction_rx: Receiver, event_tx: Sender>, } @@ -44,11 +54,12 @@ impl EventIndexer { listen_to_updates(rpc_host, rpc_auth_token, program_id, transaction_tx).await }); - tokio::spawn(async move { store_events(event_rx).await }); + let mut db_connection = establish_connection(database_connection_url); + + tokio::spawn(async move { store_events(&mut db_connection, event_rx).await }); Self { parser, - // database_connection: establish_connection(database_connection_url), transaction_rx, event_tx, } @@ -254,12 +265,69 @@ async fn listen_to_updates( } } -async fn store_events(event_rx: Receiver>) { +async fn store_events( + db_connection: &mut PgConnection, + event_rx: Receiver>, +) { loop { while let Ok(events) = event_rx.try_recv() { if !events.is_empty() { - for event in events { - info!("{:?}", event); + for MarginfiEventWithMeta { + event, + timestamp, + in_flashloan, + call_stack, + tx_sig, + } in events + { + let timestamp = DateTime::from_timestamp(timestamp, 0).unwrap().naive_utc(); + let tx_sig = tx_sig.to_string(); + let call_stack = serde_json::to_string(&call_stack).unwrap(); + + match event { + Event::CreateAccount(e) => { + let authority_id = diesel::insert_into(users::table) + .values(vec![Users { + address: e.authority.to_string(), + ..Default::default() + }]) + .on_conflict(users::id) + .do_nothing() + .returning(users::id) + .get_result(db_connection) + .unwrap(); + + let account_id = diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: e.account.to_string(), + user_id: authority_id, + ..Default::default() + }]) + .on_conflict(accounts::id) + .do_nothing() + .returning(accounts::id) + .get_result(db_connection) + .unwrap(); + + let create_account_event = CreateAccountEvents { + timestamp, + authority_id, + tx_sig, + call_stack, + in_flashloan, + account_id, + ..Default::default() + }; + + diesel::insert_into(create_account_events::table) + .values(&create_account_event) + .execute(db_connection) + .unwrap(); + } + _ => { + debug!("Unsupported event: {:?}", event); + } + } } } } diff --git a/observability/crates/event_indexer/src/lib.rs b/observability/crates/event_indexer/src/lib.rs index 2219b7e0..d2d07536 100644 --- a/observability/crates/event_indexer/src/lib.rs +++ b/observability/crates/event_indexer/src/lib.rs @@ -2,4 +2,5 @@ pub mod backfiller; pub mod error; pub mod indexer; pub mod parser; -pub mod snapshot; +// pub mod snapshot; +pub mod db; diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 8844b2a2..83d9fd4e 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -28,10 +28,9 @@ use tracing::{error, warn}; const SPL_TRANSFER_DISCRIMINATOR: u8 = 3; pub const MARGINFI_GROUP_ADDRESS: Pubkey = pubkey!("4qp6Fx6tnZkY5Wropq9wUYgtFxXKwE6viZxFHg3rdAG8"); -const COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT: u64 = 232836972; - +const COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT: u64 = 232_836_972; #[derive(Debug)] -pub enum MarginfiEvent { +pub enum Event { // User actions CreateAccount(CreateAccountEvent), AccountAuthorityTransfer(AccountAuthorityTransferEvent), @@ -50,7 +49,7 @@ pub enum MarginfiEvent { pub struct MarginfiEventWithMeta { pub timestamp: i64, pub tx_sig: Signature, - pub event: MarginfiEvent, + pub event: Event, pub in_flashloan: bool, pub call_stack: Vec, } @@ -275,7 +274,7 @@ impl MarginfiEventParser { remaining_instructions: &[InnerInstruction], account_keys: &[Pubkey], in_flashloan: &mut bool, - ) -> Option { + ) -> Option { if instruction.data.len() < 8 { error!("Instruction data too short"); return None; @@ -299,7 +298,7 @@ impl MarginfiEventParser { let marginfi_account = *ix_accounts.get(1).unwrap(); let authority = *ix_accounts.get(2).unwrap(); - Some(MarginfiEvent::CreateAccount(CreateAccountEvent { + Some(Event::CreateAccount(CreateAccountEvent { account: marginfi_account, authority, })) @@ -314,7 +313,7 @@ impl MarginfiEventParser { let signer = *ix_accounts.get(2).unwrap(); let new_authority = *ix_accounts.get(3).unwrap(); - Some(MarginfiEvent::AccountAuthorityTransfer( + Some(Event::AccountAuthorityTransfer( AccountAuthorityTransferEvent { account: marginfi_account, old_authority: signer, @@ -343,7 +342,7 @@ impl MarginfiEventParser { let signer = *ix_accounts.get(2).unwrap(); let bank = *ix_accounts.get(3).unwrap(); - Some(MarginfiEvent::Deposit(DepositEvent { + Some(Event::Deposit(DepositEvent { account: marginfi_account, authority: signer, bank, @@ -371,7 +370,7 @@ impl MarginfiEventParser { let signer = *ix_accounts.get(2).unwrap(); let bank = *ix_accounts.get(3).unwrap(); - Some(MarginfiEvent::Borrow(BorrowEvent { + Some(Event::Borrow(BorrowEvent { account: marginfi_account, authority: signer, bank, @@ -401,7 +400,7 @@ impl MarginfiEventParser { let signer = *ix_accounts.get(2).unwrap(); let bank = *ix_accounts.get(3).unwrap(); - Some(MarginfiEvent::Repay(RepayEvent { + Some(Event::Repay(RepayEvent { account: marginfi_account, authority: signer, bank, @@ -433,7 +432,7 @@ impl MarginfiEventParser { let signer = *ix_accounts.get(2).unwrap(); let bank = *ix_accounts.get(3).unwrap(); - Some(MarginfiEvent::Withdraw(WithdrawEvent { + Some(Event::Withdraw(WithdrawEvent { account: marginfi_account, authority: signer, bank, @@ -456,7 +455,7 @@ impl MarginfiEventParser { let liquidator_authority = *ix_accounts.get(4).unwrap(); let liquidatee_account = *ix_accounts.get(5).unwrap(); - Some(MarginfiEvent::Liquidate(LiquidateEvent { + Some(Event::Liquidate(LiquidateEvent { asset_amount: instruction.asset_amount, asset_bank, liability_bank, @@ -483,7 +482,7 @@ impl MarginfiEventParser { let bank = *ix_accounts.get(3).unwrap(); let emissions_mint = *ix_accounts.get(4).unwrap(); - Some(MarginfiEvent::WithdrawEmissions(WithdrawEmissionsEvent { + Some(Event::WithdrawEmissions(WithdrawEmissionsEvent { account: marginfi_account, authority: signer, bank, @@ -508,7 +507,7 @@ impl MarginfiEventParser { let bank_mint = *ix_accounts.get(3).unwrap(); let bank = *ix_accounts.get(4).unwrap(); - Some(MarginfiEvent::AddBank(AddBankEvent { + Some(Event::AddBank(AddBankEvent { bank, mint: bank_mint, config: bank_config, From 262982179064840127ee6bad06440f46be58d02e Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:09:45 +0800 Subject: [PATCH 03/20] feat: wip db storage --- observability/Cargo.lock | 1 + observability/Cargo.toml | 1 + observability/crates/event_indexer/Cargo.toml | 1 + .../2024-03-14-151141_initial_tables/up.sql | 57 +- .../crates/event_indexer/src/db/models.rs | 34 +- .../crates/event_indexer/src/db/schema.rs | 29 +- .../crates/event_indexer/src/entity_store.rs | 257 +++++ .../crates/event_indexer/src/error.rs | 6 + .../crates/event_indexer/src/indexer.rs | 89 +- observability/crates/event_indexer/src/lib.rs | 5 +- .../crates/event_indexer/src/parser.rs | 1001 +++++++++++++++-- 11 files changed, 1311 insertions(+), 170 deletions(-) create mode 100644 observability/crates/event_indexer/src/entity_store.rs diff --git a/observability/Cargo.lock b/observability/Cargo.lock index cea5538e..4b40bebf 100644 --- a/observability/Cargo.lock +++ b/observability/Cargo.lock @@ -1611,6 +1611,7 @@ dependencies = [ "crossbeam", "diesel", "dotenv", + "enum_dispatch", "envconfig", "futures", "marginfi", diff --git a/observability/Cargo.toml b/observability/Cargo.toml index 6a77238f..5faa2ffd 100644 --- a/observability/Cargo.toml +++ b/observability/Cargo.toml @@ -13,6 +13,7 @@ diesel = { git = "https://github.com/diesel-rs/diesel.git", rev = "12429cb4eebfb dotenv = "0.15.0" futures = "0.3.25" envconfig = "0.10.0" +enum_dispatch = "0.3.12" marginfi = { path = "../programs/marginfi", features = [ "no-entrypoint", "client", diff --git a/observability/crates/event_indexer/Cargo.toml b/observability/crates/event_indexer/Cargo.toml index 2958757a..3429e5a5 100644 --- a/observability/crates/event_indexer/Cargo.toml +++ b/observability/crates/event_indexer/Cargo.toml @@ -23,6 +23,7 @@ chrono = { workspace = true } crossbeam = { workspace = true } diesel = { workspace = true, features = ["postgres", "chrono", "numeric"] } dotenv = { workspace = true } +enum_dispatch = { workspace = true } envconfig = { workspace = true } futures = { workspace = true } marginfi = { workspace = true } diff --git a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql index e974eefb..cffe6236 100644 --- a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql +++ b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql @@ -4,25 +4,37 @@ CREATE TABLE "mints"( "id" SERIAL PRIMARY KEY, "address" VARCHAR NOT NULL, "symbol" VARCHAR NOT NULL, - "decimals" SMALLINT NOT NULL + "decimals" SMALLINT NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('mints'); CREATE TABLE "banks"( "id" SERIAL PRIMARY KEY, "address" VARCHAR NOT NULL UNIQUE, - "mint" INT4 NOT NULL REFERENCES "mints"("id") + "mint_id" INT4 NOT NULL REFERENCES "mints"("id"), + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('banks'); CREATE TABLE "users"( "id" SERIAL PRIMARY KEY, - "address" VARCHAR NOT NULL UNIQUE + "address" VARCHAR NOT NULL UNIQUE, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('users'); CREATE TABLE "accounts"( "id" SERIAL PRIMARY KEY, "address" VARCHAR NOT NULL UNIQUE, - "user_id" INT4 NOT NULL REFERENCES "users"("id") + "user_id" INT4 NOT NULL REFERENCES "users"("id"), + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('accounts'); CREATE TABLE "create_account_events"( "id" SERIAL PRIMARY KEY, @@ -31,8 +43,11 @@ CREATE TABLE "create_account_events"( "in_flashloan" BOOLEAN NOT NULL, "call_stack" VARCHAR NOT NULL, "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), - "authority_id" INT4 NOT NULL REFERENCES "users"("id") + "authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('create_account_events'); CREATE TABLE "transfer_account_authority_events"( "id" SERIAL PRIMARY KEY, @@ -42,8 +57,11 @@ CREATE TABLE "transfer_account_authority_events"( "call_stack" VARCHAR NOT NULL, "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "old_authority_id" INT4 NOT NULL REFERENCES "users"("id"), - "new_authority_id" INT4 NOT NULL REFERENCES "users"("id") + "new_authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('transfer_account_authority_events'); CREATE TABLE "deposit_events"( "id" SERIAL PRIMARY KEY, @@ -54,8 +72,11 @@ CREATE TABLE "deposit_events"( "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), - "amount" NUMERIC NOT NULL + "amount" NUMERIC NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('deposit_events'); CREATE TABLE "borrow_events"( "id" SERIAL PRIMARY KEY, @@ -66,8 +87,11 @@ CREATE TABLE "borrow_events"( "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), - "amount" NUMERIC NOT NULL + "amount" NUMERIC NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('borrow_events'); CREATE TABLE "repay_events"( "id" SERIAL PRIMARY KEY, @@ -79,8 +103,11 @@ CREATE TABLE "repay_events"( "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, - "all" BOOLEAN NOT NULL + "all" BOOLEAN NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('repay_events'); CREATE TABLE "withdraw_events"( "id" SERIAL PRIMARY KEY, @@ -92,8 +119,11 @@ CREATE TABLE "withdraw_events"( "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, - "all" BOOLEAN NOT NULL + "all" BOOLEAN NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('withdraw_events'); CREATE TABLE "withdraw_emissions_events"( "id" SERIAL PRIMARY KEY, @@ -104,6 +134,9 @@ CREATE TABLE "withdraw_emissions_events"( "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), - "emission_mint" VARCHAR NOT NULL, - "amount" NUMERIC NOT NULL + "emission_mint_id" INT4 NOT NULL REFERENCES "mints"("id"), + "amount" NUMERIC NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); +SELECT diesel_manage_updated_at('withdraw_emissions_events'); diff --git a/observability/crates/event_indexer/src/db/models.rs b/observability/crates/event_indexer/src/db/models.rs index 89ed7414..00a8ad8e 100644 --- a/observability/crates/event_indexer/src/db/models.rs +++ b/observability/crates/event_indexer/src/db/models.rs @@ -2,16 +2,26 @@ use crate::db::schema::*; use diesel::prelude::*; use rust_decimal::Decimal; -#[derive(Default, Queryable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable)] +#[diesel(table_name = mints)] +pub struct Mints { + #[diesel(skip_insertion)] + pub id: i32, + pub address: String, + pub symbol: String, + pub decimals: i16, +} + +#[derive(Default, Debug, Queryable, Selectable, Insertable)] #[diesel(table_name = banks)] pub struct Banks { #[diesel(skip_insertion)] pub id: i32, pub address: String, - pub mint: i32, + pub mint_id: i32, } -#[derive(Default, Queryable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable)] #[diesel(table_name = users)] pub struct Users { #[diesel(skip_insertion)] @@ -19,7 +29,7 @@ pub struct Users { pub address: String, } -#[derive(Default, Queryable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable)] #[diesel(table_name = accounts)] pub struct Accounts { #[diesel(skip_insertion)] @@ -28,7 +38,7 @@ pub struct Accounts { pub user_id: i32, } -#[derive(Default, Queryable, Insertable, Associations)] +#[derive(Default, Debug, Queryable, Selectable, Insertable, Associations)] #[diesel(belongs_to(Accounts, foreign_key = account_id), belongs_to(Users, foreign_key = authority_id))] #[diesel(table_name = create_account_events)] pub struct CreateAccountEvents { @@ -42,7 +52,7 @@ pub struct CreateAccountEvents { pub authority_id: i32, } -#[derive(Default, Queryable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable)] #[diesel(table_name = transfer_account_authority_events)] pub struct TransferAccountAuthorityEvents { #[diesel(skip_insertion)] @@ -56,7 +66,7 @@ pub struct TransferAccountAuthorityEvents { pub new_authority_id: i32, } -#[derive(Default, Queryable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable)] #[diesel(table_name = deposit_events)] pub struct DepositEvents { #[diesel(skip_insertion)] @@ -71,7 +81,7 @@ pub struct DepositEvents { pub amount: Decimal, } -#[derive(Default, Queryable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable)] #[diesel(table_name = borrow_events)] pub struct BorrowEvents { #[diesel(skip_insertion)] @@ -86,7 +96,7 @@ pub struct BorrowEvents { pub amount: Decimal, } -#[derive(Default, Queryable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable)] #[diesel(table_name = repay_events)] pub struct RepayEvents { #[diesel(skip_insertion)] @@ -102,7 +112,7 @@ pub struct RepayEvents { pub all: bool, } -#[derive(Default, Queryable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable)] #[diesel(table_name = withdraw_events)] pub struct WithdrawEvents { #[diesel(skip_insertion)] @@ -118,7 +128,7 @@ pub struct WithdrawEvents { pub all: bool, } -#[derive(Default, Queryable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable)] #[diesel(table_name = withdraw_emissions_events)] pub struct WithdrawEmissionsEvents { #[diesel(skip_insertion)] @@ -130,6 +140,6 @@ pub struct WithdrawEmissionsEvents { pub account_id: i32, pub authority_id: i32, pub bank_id: i32, - pub emission_mint: String, + pub emission_mint_id: i32, pub amount: Decimal, } diff --git a/observability/crates/event_indexer/src/db/schema.rs b/observability/crates/event_indexer/src/db/schema.rs index 94708b47..5b7160fe 100644 --- a/observability/crates/event_indexer/src/db/schema.rs +++ b/observability/crates/event_indexer/src/db/schema.rs @@ -5,6 +5,8 @@ diesel::table! { id -> Int4, address -> Varchar, user_id -> Int4, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -12,7 +14,9 @@ diesel::table! { banks (id) { id -> Int4, address -> Varchar, - mint -> Int4, + mint_id -> Int4, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -27,6 +31,8 @@ diesel::table! { authority_id -> Int4, bank_id -> Int4, amount -> Numeric, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -39,6 +45,8 @@ diesel::table! { call_stack -> Varchar, account_id -> Int4, authority_id -> Int4, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -53,6 +61,8 @@ diesel::table! { authority_id -> Int4, bank_id -> Int4, amount -> Numeric, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -62,6 +72,8 @@ diesel::table! { address -> Varchar, symbol -> Varchar, decimals -> Int2, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -77,6 +89,8 @@ diesel::table! { bank_id -> Int4, amount -> Numeric, all -> Bool, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -90,6 +104,8 @@ diesel::table! { account_id -> Int4, old_authority_id -> Int4, new_authority_id -> Int4, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -97,6 +113,8 @@ diesel::table! { users (id) { id -> Int4, address -> Varchar, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -110,8 +128,10 @@ diesel::table! { account_id -> Int4, authority_id -> Int4, bank_id -> Int4, - emission_mint -> Varchar, + emission_mint_id -> Int4, amount -> Numeric, + created_at -> Timestamp, + updated_at -> Timestamp, } } @@ -127,11 +147,13 @@ diesel::table! { bank_id -> Int4, amount -> Numeric, all -> Bool, + created_at -> Timestamp, + updated_at -> Timestamp, } } diesel::joinable!(accounts -> users (user_id)); -diesel::joinable!(banks -> mints (mint)); +diesel::joinable!(banks -> mints (mint_id)); diesel::joinable!(borrow_events -> accounts (account_id)); diesel::joinable!(borrow_events -> banks (bank_id)); diesel::joinable!(borrow_events -> users (authority_id)); @@ -146,6 +168,7 @@ diesel::joinable!(repay_events -> users (authority_id)); diesel::joinable!(transfer_account_authority_events -> accounts (account_id)); diesel::joinable!(withdraw_emissions_events -> accounts (account_id)); diesel::joinable!(withdraw_emissions_events -> banks (bank_id)); +diesel::joinable!(withdraw_emissions_events -> mints (emission_mint_id)); diesel::joinable!(withdraw_emissions_events -> users (authority_id)); diesel::joinable!(withdraw_events -> accounts (account_id)); diesel::joinable!(withdraw_events -> banks (bank_id)); diff --git a/observability/crates/event_indexer/src/entity_store.rs b/observability/crates/event_indexer/src/entity_store.rs new file mode 100644 index 00000000..9373403c --- /dev/null +++ b/observability/crates/event_indexer/src/entity_store.rs @@ -0,0 +1,257 @@ +use std::{collections::HashMap, str::FromStr}; + +use anchor_lang::AccountDeserialize; +use diesel::{ExpressionMethods, PgConnection, QueryDsl, RunQueryDsl, SelectableHelper}; +use marginfi::state::marginfi_group::Bank; +use solana_client::rpc_client::RpcClient; +use solana_sdk::{ + commitment_config::{CommitmentConfig, CommitmentLevel}, + program_pack::Pack, + pubkey::Pubkey, +}; +use spl_token::state::Mint; + +use crate::{ + db::{establish_connection, models::*, schema::*}, + error::IndexingError, +}; + +pub struct EntityStore { + pub rpc_client: RpcClient, + pub db_connection: PgConnection, + mint_cache: HashMap, + bank_cache: HashMap, + account_cache: HashMap, + authority_cache: HashMap, +} + +impl EntityStore { + pub fn new(rpc_endpoint: String, db_connection_url: String) -> Self { + let rpc_client = RpcClient::new_with_commitment( + rpc_endpoint, + CommitmentConfig { + commitment: CommitmentLevel::Confirmed, + }, + ); + + let db_connection = establish_connection(db_connection_url); + + EntityStore { + rpc_client, + db_connection, + mint_cache: HashMap::new(), + bank_cache: HashMap::new(), + account_cache: HashMap::new(), + authority_cache: HashMap::new(), + } + } + + pub fn get_or_fetch_mint(&mut self, address: &str) -> Result { + let maybe_mint = self.mint_cache.get(&address.to_string()); + + if let Some(mint) = maybe_mint { + Ok(mint.clone()) + } else { + let mint = MintData::fetch(&self.rpc_client, &mut self.db_connection, address)?; + Ok(mint) + } + } + + pub fn get_or_fetch_bank(&mut self, address: &str) -> Result { + let maybe_bank = self.bank_cache.get(&address.to_string()); + + if let Some(bank) = maybe_bank { + Ok(bank.clone()) + } else { + let bank = BankData::fetch(&self.rpc_client, &mut self.db_connection, address)?; + Ok(bank) + } + } +} + +#[derive(Debug, Clone)] +pub struct MintData { + pub id: Option, + pub address: String, + pub symbol: String, + pub decimals: i16, +} + +impl MintData { + pub fn fetch( + rpc_client: &RpcClient, + db_connection: &mut PgConnection, + address: &str, + ) -> Result { + let db_record = MintData::fetch_from_db(db_connection, address)?; + + if let Some(db_record) = db_record { + return Ok(db_record); + } + + let mint = MintData::fetch_from_rpc(rpc_client, address)?; + if let Some(mint) = mint { + Ok(mint) + } else { + Err(IndexingError::FailedToFetchEntity(format!( + "Mint not found: {}", + address + ))) + } + } + + fn fetch_from_db( + db_connection: &mut PgConnection, + address: &str, + ) -> Result, IndexingError> { + let db_records = mints::dsl::mints + .filter(mints::address.eq(address.to_string())) + .select(Mints::as_select()) + .limit(1) + .load(db_connection) + .map_err(|e| IndexingError::FailedToFetchEntity(e.to_string()))?; + + if db_records.is_empty() { + return Ok(None); + } + + let db_record = db_records.get(0).unwrap(); + + Ok(Some(MintData { + id: Some(db_record.id), + address: db_record.address.clone(), + symbol: db_record.symbol.clone(), + decimals: db_record.decimals, + })) + } + + fn fetch_from_rpc( + rpc_client: &RpcClient, + address: &str, + ) -> Result, IndexingError> { + let mint_data = rpc_client + .get_account_data(&Pubkey::from_str(address).unwrap()) + .map_err(|e| IndexingError::FailedToFetchEntity(e.to_string()))?; + + let mint = Mint::unpack_from_slice(&mint_data).map_err(|e| { + IndexingError::FailedToFetchEntity(format!("Failed to unpack mint: {}", e)) + })?; + + Ok(Some(MintData { + id: None, + address: address.to_string(), + symbol: "".to_string(), + decimals: mint.decimals as i16, + })) + } +} + +#[derive(Debug, Clone)] +pub struct BankData { + pub id: Option, + pub address: String, + pub mint: MintData, +} + +impl BankData { + pub fn fetch( + rpc_client: &RpcClient, + db_connection: &mut PgConnection, + address: &str, + ) -> Result { + let db_record = BankData::fetch_from_db(db_connection, address)?; + + if let Some(db_record) = db_record { + return Ok(db_record); + } + + let mint = BankData::fetch_from_rpc(rpc_client, db_connection, address)?; + if let Some(mint) = mint { + Ok(mint) + } else { + Err(IndexingError::FailedToFetchEntity(format!( + "Bank not found: {}", + address + ))) + } + } + + fn fetch_from_db( + db_connection: &mut PgConnection, + address: &str, + ) -> Result, IndexingError> { + let db_records = banks::dsl::banks + .filter(banks::address.eq(address.to_string())) + .select(Banks::as_select()) + .limit(1) + .load(db_connection) + .map_err(|e| IndexingError::FailedToFetchEntity(e.to_string()))?; + + if db_records.is_empty() { + return Ok(None); + } + + let db_record = db_records.get(0).unwrap(); + + let mint_data = MintData::fetch_from_db(db_connection, &db_record.mint_id.to_string())? + .ok_or_else(|| { + IndexingError::FailedToFetchEntity(format!( + "Mint not found in DB: {}", + db_record.mint_id + )) + })?; + + Ok(Some(BankData { + id: Some(db_record.id), + address: db_record.address.clone(), + mint: mint_data, + })) + } + + fn fetch_from_rpc( + rpc_client: &RpcClient, + db_connection: &mut PgConnection, + address: &str, + ) -> Result, IndexingError> { + let mint_data = rpc_client + .get_account_data(&Pubkey::from_str(address).unwrap()) + .map_err(|e| IndexingError::FailedToFetchEntity(e.to_string()))?; + + let bank = Bank::try_deserialize(&mut mint_data.as_slice()).map_err(|e| { + IndexingError::FailedToFetchEntity(format!("Failed to unpack bank: {}", e)) + })?; + + let mint_data = MintData::fetch(rpc_client, db_connection, &bank.mint.to_string())?; + + Ok(Some(BankData { + id: None, + address: address.to_string(), + mint: mint_data, + })) + } +} + +#[derive(Debug, Clone)] +pub struct AccountData { + pub id: Option, + pub address: String, + pub user_id: i32, +} + +impl AccountData { + fn get_cache_or_fetch(&mut self, address: &str) -> AccountData { + todo!() + } +} + +#[derive(Debug, Clone)] +pub struct UserData { + pub id: Option, + pub address: String, +} + +impl UserData { + fn get_cache_or_fetch(&mut self, address: &str) -> UserData { + todo!() + } +} diff --git a/observability/crates/event_indexer/src/error.rs b/observability/crates/event_indexer/src/error.rs index 6af9eb97..7c8b1100 100644 --- a/observability/crates/event_indexer/src/error.rs +++ b/observability/crates/event_indexer/src/error.rs @@ -15,6 +15,12 @@ pub enum IndexingError { #[error("Failed to find slot for tx signature {0:?}")] FailedToFindTransactionSlot(Signature), + #[error("Failed to fetch entity: {0}")] + FailedToFetchEntity(String), + + #[error("Failed to insert event: {0}")] + FailedToInsertEvent(String), + #[error("An unknown error occurred")] Unknown, } diff --git a/observability/crates/event_indexer/src/indexer.rs b/observability/crates/event_indexer/src/indexer.rs index e5e7ea82..dfa30026 100644 --- a/observability/crates/event_indexer/src/indexer.rs +++ b/observability/crates/event_indexer/src/indexer.rs @@ -7,12 +7,12 @@ use std::{ use chrono::DateTime; use crossbeam::channel::{Receiver, Sender}; -use diesel::{PgConnection, RunQueryDsl}; +use diesel::PgConnection; use futures::StreamExt; use solana_client::rpc_client::SerializableTransaction; use solana_sdk::{pubkey::Pubkey, signature::Signature}; use solana_transaction_status::{TransactionWithStatusMeta, VersionedTransactionWithStatusMeta}; -use tracing::{debug, error, info, warn}; +use tracing::{error, info, warn}; use yellowstone_grpc_client::GeyserGrpcClient; use yellowstone_grpc_proto::{ convert_from, @@ -22,14 +22,7 @@ use yellowstone_grpc_proto::{ }, }; -use crate::{ - db::{ - establish_connection, - models::{Accounts, CreateAccountEvents, Users}, - schema::*, - }, - parser::Event, -}; +use crate::{db::establish_connection, entity_store::EntityStore, parser::MarginfiEvent}; use super::parser::{MarginfiEventParser, MarginfiEventWithMeta, MARGINFI_GROUP_ADDRESS}; @@ -50,13 +43,26 @@ impl EventIndexer { let (transaction_tx, transaction_rx) = crossbeam::channel::unbounded::(); let (event_tx, event_rx) = crossbeam::channel::unbounded::>(); + let rpc_host_clone = rpc_host.clone(); + let rpc_auth_token_clone = rpc_auth_token.clone(); tokio::spawn(async move { - listen_to_updates(rpc_host, rpc_auth_token, program_id, transaction_tx).await + listen_to_updates( + rpc_host_clone, + rpc_auth_token_clone, + program_id, + transaction_tx, + ) + .await }); - let mut db_connection = establish_connection(database_connection_url); + let mut db_connection = establish_connection(database_connection_url.clone()); + + let rpc_endpoint = format!("{}/{}", rpc_host, rpc_auth_token).to_string(); + let mut entity_store = EntityStore::new(rpc_endpoint, database_connection_url); - tokio::spawn(async move { store_events(&mut db_connection, event_rx).await }); + tokio::spawn( + async move { store_events(&mut db_connection, event_rx, &mut entity_store).await }, + ); Self { parser, @@ -268,6 +274,7 @@ async fn listen_to_updates( async fn store_events( db_connection: &mut PgConnection, event_rx: Receiver>, + entity_store: &mut EntityStore, ) { loop { while let Ok(events) = event_rx.try_recv() { @@ -283,51 +290,17 @@ async fn store_events( let timestamp = DateTime::from_timestamp(timestamp, 0).unwrap().naive_utc(); let tx_sig = tx_sig.to_string(); let call_stack = serde_json::to_string(&call_stack).unwrap(); - - match event { - Event::CreateAccount(e) => { - let authority_id = diesel::insert_into(users::table) - .values(vec![Users { - address: e.authority.to_string(), - ..Default::default() - }]) - .on_conflict(users::id) - .do_nothing() - .returning(users::id) - .get_result(db_connection) - .unwrap(); - - let account_id = diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: e.account.to_string(), - user_id: authority_id, - ..Default::default() - }]) - .on_conflict(accounts::id) - .do_nothing() - .returning(accounts::id) - .get_result(db_connection) - .unwrap(); - - let create_account_event = CreateAccountEvents { - timestamp, - authority_id, - tx_sig, - call_stack, - in_flashloan, - account_id, - ..Default::default() - }; - - diesel::insert_into(create_account_events::table) - .values(&create_account_event) - .execute(db_connection) - .unwrap(); - } - _ => { - debug!("Unsupported event: {:?}", event); - } - } + println!("Storing event: {:?}", event); + event + .db_insert( + timestamp, + tx_sig, + in_flashloan, + call_stack, + db_connection, + entity_store, + ) + .unwrap(); } } } diff --git a/observability/crates/event_indexer/src/lib.rs b/observability/crates/event_indexer/src/lib.rs index d2d07536..1c9fc704 100644 --- a/observability/crates/event_indexer/src/lib.rs +++ b/observability/crates/event_indexer/src/lib.rs @@ -1,6 +1,7 @@ pub mod backfiller; +pub mod db; +pub mod entity_store; pub mod error; pub mod indexer; pub mod parser; -// pub mod snapshot; -pub mod db; +pub mod snapshot; diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 83d9fd4e..1fdb303a 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -1,17 +1,19 @@ use std::collections::HashMap; use anchor_lang::{AnchorDeserialize, Discriminator}; -use marginfi::{ - instruction::{ - LendingAccountBorrow, LendingAccountCloseBalance, LendingAccountDeposit, - LendingAccountEndFlashloan, LendingAccountLiquidate, LendingAccountRepay, - LendingAccountSettleEmissions, LendingAccountStartFlashloan, LendingAccountWithdraw, - LendingAccountWithdrawEmissions, LendingPoolAccrueBankInterest, LendingPoolAddBank, - LendingPoolAddBankWithSeed, LendingPoolConfigureBank, MarginfiAccountInitialize, - SetNewAccountAuthority, - }, - state::marginfi_group::{BankConfig, BankConfigCompact}, +use chrono::NaiveDateTime; +use diesel::{ + Connection, ExpressionMethods, PgConnection, QueryDsl, RunQueryDsl, SelectableHelper, }; +use enum_dispatch::enum_dispatch; +use marginfi::instruction::{ + LendingAccountBorrow, LendingAccountCloseBalance, LendingAccountDeposit, + LendingAccountEndFlashloan, LendingAccountRepay, LendingAccountSettleEmissions, + LendingAccountStartFlashloan, LendingAccountWithdraw, LendingAccountWithdrawEmissions, + LendingPoolAccrueBankInterest, LendingPoolAddBankWithSeed, LendingPoolConfigureBank, + MarginfiAccountInitialize, SetNewAccountAuthority, +}; +use rust_decimal::{prelude::FromPrimitive, Decimal}; use solana_sdk::{ hash::Hash, instruction::CompiledInstruction, @@ -26,9 +28,39 @@ use solana_transaction_status::{ }; use tracing::{error, warn}; +use crate::{ + db::{models::*, schema::*}, + entity_store::EntityStore, + error::IndexingError, +}; + const SPL_TRANSFER_DISCRIMINATOR: u8 = 3; pub const MARGINFI_GROUP_ADDRESS: Pubkey = pubkey!("4qp6Fx6tnZkY5Wropq9wUYgtFxXKwE6viZxFHg3rdAG8"); const COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT: u64 = 232_836_972; + +#[derive(Debug)] +pub struct MarginfiEventWithMeta { + pub timestamp: i64, + pub tx_sig: Signature, + pub event: Event, + pub in_flashloan: bool, + pub call_stack: Vec, +} + +#[enum_dispatch] +pub trait MarginfiEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError>; +} + +#[enum_dispatch(MarginfiEvent)] #[derive(Debug)] pub enum Event { // User actions @@ -39,19 +71,10 @@ pub enum Event { Repay(RepayEvent), Withdraw(WithdrawEvent), WithdrawEmissions(WithdrawEmissionsEvent), - Liquidate(LiquidateEvent), + // Liquidate(LiquidateEvent), - // Admin actions - AddBank(AddBankEvent), -} - -#[derive(Debug)] -pub struct MarginfiEventWithMeta { - pub timestamp: i64, - pub tx_sig: Signature, - pub event: Event, - pub in_flashloan: bool, - pub call_stack: Vec, + // // Admin actions + // AddBank(AddBankEvent), } #[derive(Debug)] @@ -60,6 +83,77 @@ pub struct CreateAccountEvent { pub authority: Pubkey, } +impl MarginfiEvent for CreateAccountEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + db_connection + .transaction(|connection: &mut PgConnection| { + let users = users::dsl::users + .filter(users::address.eq(self.authority.to_string())) + .select(Users::as_select()) + .limit(1) + .load(connection)?; + + let authority_id = if users.len() == 0 { + diesel::insert_into(users::table) + .values(vec![Users { + address: self.authority.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(users::id) + .get_result(connection)? + } else { + users.first().unwrap().id + }; + + let accounts = accounts::dsl::accounts + .filter(accounts::address.eq(self.account.to_string())) + .select(Accounts::as_select()) + .limit(1) + .load(connection)?; + + let account_id = if accounts.len() == 0 { + diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(accounts::id) + .get_result(connection)? + } else { + accounts.first().unwrap().id + }; + + let create_account_event = CreateAccountEvents { + timestamp, + authority_id, + tx_sig, + call_stack, + in_flashloan, + account_id, + ..Default::default() + }; + + diesel::insert_into(create_account_events::table) + .values(&create_account_event) + .execute(connection)?; + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } +} + #[derive(Debug)] pub struct AccountAuthorityTransferEvent { pub account: Pubkey, @@ -67,6 +161,97 @@ pub struct AccountAuthorityTransferEvent { pub new_authority: Pubkey, } +impl MarginfiEvent for AccountAuthorityTransferEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + db_connection + .transaction(|connection: &mut PgConnection| { + let users = users::dsl::users + .filter(users::address.eq(self.old_authority.to_string())) + .select(Users::as_select()) + .limit(1) + .load(connection)?; + + let old_authority_id = if users.len() == 0 { + diesel::insert_into(users::table) + .values(vec![Users { + address: self.old_authority.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(users::id) + .get_result(connection)? + } else { + users.first().unwrap().id + }; + + let users = users::dsl::users + .filter(users::address.eq(self.new_authority.to_string())) + .select(Users::as_select()) + .limit(1) + .load(connection)?; + + let new_authority_id = if users.len() == 0 { + diesel::insert_into(users::table) + .values(vec![Users { + address: self.new_authority.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(users::id) + .get_result(connection)? + } else { + users.first().unwrap().id + }; + + let accounts = accounts::dsl::accounts + .filter(accounts::address.eq(self.account.to_string())) + .select(Accounts::as_select()) + .limit(1) + .load(connection)?; + + let account_id = if accounts.len() == 0 { + diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: self.account.to_string(), + user_id: new_authority_id, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(accounts::id) + .get_result(connection)? + } else { + accounts.first().unwrap().id + }; + + let account_authority_transfer_event = TransferAccountAuthorityEvents { + timestamp, + old_authority_id, + new_authority_id, + tx_sig, + call_stack, + in_flashloan, + account_id, + ..Default::default() + }; + + diesel::insert_into(transfer_account_authority_events::table) + .values(&account_authority_transfer_event) + .execute(connection)?; + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } +} + #[derive(Debug)] pub struct DepositEvent { pub account: Pubkey, @@ -75,6 +260,124 @@ pub struct DepositEvent { pub amount: u64, } +impl MarginfiEvent for DepositEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + let bank_data = entity_store + .get_or_fetch_bank(&self.bank.to_string()) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + + db_connection + .transaction(|connection: &mut PgConnection| { + let users = users::dsl::users + .filter(users::address.eq(self.authority.to_string())) + .select(Users::as_select()) + .limit(1) + .load(connection)?; + + let authority_id = if users.len() == 0 { + diesel::insert_into(users::table) + .values(vec![Users { + address: self.authority.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(users::id) + .get_result(connection)? + } else { + users.first().unwrap().id + }; + + let accounts = accounts::dsl::accounts + .filter(accounts::address.eq(self.account.to_string())) + .select(Accounts::as_select()) + .limit(1) + .load(connection)?; + + let account_id = if accounts.len() == 0 { + diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(accounts::id) + .get_result(connection)? + } else { + accounts.first().unwrap().id + }; + + let mints = mints::dsl::mints + .filter(mints::address.eq(bank_data.mint.address.clone())) + .select(Mints::as_select()) + .limit(1) + .load(connection)?; + + let mint_id = if mints.len() == 0 { + diesel::insert_into(mints::table) + .values(vec![Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(mints::id) + .get_result(connection)? + } else { + mints.first().unwrap().id + }; + + let banks = banks::dsl::banks + .filter(banks::address.eq(self.bank.to_string())) + .select(Banks::as_select()) + .limit(1) + .load(connection)?; + + let bank_id = if banks.len() == 0 { + diesel::insert_into(banks::table) + .values(vec![Banks { + address: self.bank.to_string(), + mint_id, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(banks::id) + .get_result(connection)? + } else { + banks.first().unwrap().id + }; + + let deposit_event = DepositEvents { + timestamp, + authority_id, + tx_sig, + call_stack, + in_flashloan, + account_id, + bank_id, + amount: Decimal::from_u64(self.amount).unwrap(), + ..Default::default() + }; + + diesel::insert_into(deposit_events::table) + .values(&deposit_event) + .execute(connection)?; + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } +} + #[derive(Debug)] pub struct BorrowEvent { pub account: Pubkey, @@ -83,6 +386,98 @@ pub struct BorrowEvent { pub amount: u64, } +impl MarginfiEvent for BorrowEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + db_connection + .transaction(|connection: &mut PgConnection| { + let users = users::dsl::users + .filter(users::address.eq(self.authority.to_string())) + .select(Users::as_select()) + .limit(1) + .load(connection)?; + + let authority_id = if users.len() == 0 { + diesel::insert_into(users::table) + .values(vec![Users { + address: self.authority.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(users::id) + .get_result(connection)? + } else { + users.first().unwrap().id + }; + + let accounts = accounts::dsl::accounts + .filter(accounts::address.eq(self.account.to_string())) + .select(Accounts::as_select()) + .limit(1) + .load(connection)?; + + let account_id = if accounts.len() == 0 { + diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(accounts::id) + .get_result(connection)? + } else { + accounts.first().unwrap().id + }; + + let banks = banks::dsl::banks + .filter(banks::address.eq(self.bank.to_string())) + .select(Banks::as_select()) + .limit(1) + .load(connection)?; + + let bank_id = if banks.len() == 0 { + diesel::insert_into(banks::table) + .values(vec![Banks { + address: self.bank.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(banks::id) + .get_result(connection)? + } else { + banks.first().unwrap().id + }; + + let borrow_event = BorrowEvents { + timestamp, + authority_id, + tx_sig, + call_stack, + in_flashloan, + account_id, + bank_id, + amount: Decimal::from_u64(self.amount).unwrap(), + ..Default::default() + }; + + diesel::insert_into(borrow_events::table) + .values(&borrow_event) + .execute(connection)?; + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } +} + #[derive(Debug)] pub struct RepayEvent { pub account: Pubkey, @@ -92,6 +487,99 @@ pub struct RepayEvent { pub all: bool, } +impl MarginfiEvent for RepayEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + db_connection + .transaction(|connection: &mut PgConnection| { + let users = users::dsl::users + .filter(users::address.eq(self.authority.to_string())) + .select(Users::as_select()) + .limit(1) + .load(connection)?; + + let authority_id = if users.len() == 0 { + diesel::insert_into(users::table) + .values(vec![Users { + address: self.authority.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(users::id) + .get_result(connection)? + } else { + users.first().unwrap().id + }; + + let accounts = accounts::dsl::accounts + .filter(accounts::address.eq(self.account.to_string())) + .select(Accounts::as_select()) + .limit(1) + .load(connection)?; + + let account_id = if accounts.len() == 0 { + diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(accounts::id) + .get_result(connection)? + } else { + accounts.first().unwrap().id + }; + + let banks = banks::dsl::banks + .filter(banks::address.eq(self.bank.to_string())) + .select(Banks::as_select()) + .limit(1) + .load(connection)?; + + let bank_id = if banks.len() == 0 { + diesel::insert_into(banks::table) + .values(vec![Banks { + address: self.bank.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(banks::id) + .get_result(connection)? + } else { + banks.first().unwrap().id + }; + + let repay_event = RepayEvents { + timestamp, + authority_id, + tx_sig, + call_stack, + in_flashloan, + account_id, + bank_id, + amount: Decimal::from_u64(self.amount).unwrap(), + all: self.all, + ..Default::default() + }; + + diesel::insert_into(repay_events::table) + .values(&repay_event) + .execute(connection)?; + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } +} + #[derive(Debug)] pub struct WithdrawEvent { pub account: Pubkey, @@ -101,6 +589,99 @@ pub struct WithdrawEvent { pub all: bool, } +impl MarginfiEvent for WithdrawEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + db_connection + .transaction(|connection: &mut PgConnection| { + let users = users::dsl::users + .filter(users::address.eq(self.authority.to_string())) + .select(Users::as_select()) + .limit(1) + .load(connection)?; + + let authority_id = if users.len() == 0 { + diesel::insert_into(users::table) + .values(vec![Users { + address: self.authority.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(users::id) + .get_result(connection)? + } else { + users.first().unwrap().id + }; + + let accounts = accounts::dsl::accounts + .filter(accounts::address.eq(self.account.to_string())) + .select(Accounts::as_select()) + .limit(1) + .load(connection)?; + + let account_id = if accounts.len() == 0 { + diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(accounts::id) + .get_result(connection)? + } else { + accounts.first().unwrap().id + }; + + let banks = banks::dsl::banks + .filter(banks::address.eq(self.bank.to_string())) + .select(Banks::as_select()) + .limit(1) + .load(connection)?; + + let bank_id = if banks.len() == 0 { + diesel::insert_into(banks::table) + .values(vec![Banks { + address: self.bank.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(banks::id) + .get_result(connection)? + } else { + banks.first().unwrap().id + }; + + let withdraw_event = WithdrawEvents { + timestamp, + authority_id, + tx_sig, + call_stack, + in_flashloan, + account_id, + bank_id, + amount: Decimal::from_u64(self.amount).unwrap(), + all: self.all, + ..Default::default() + }; + + diesel::insert_into(withdraw_events::table) + .values(&withdraw_event) + .execute(connection)?; + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } +} + #[derive(Debug)] pub struct WithdrawEmissionsEvent { pub account: Pubkey, @@ -110,23 +691,277 @@ pub struct WithdrawEmissionsEvent { pub amount: u64, } -#[derive(Debug)] -pub struct LiquidateEvent { - pub asset_amount: u64, - pub asset_bank: Pubkey, - pub liability_bank: Pubkey, - pub liquidator_account: Pubkey, - pub liquidator_authority: Pubkey, - pub liquidatee_account: Pubkey, -} +impl MarginfiEvent for WithdrawEmissionsEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + db_connection + .transaction(|connection: &mut PgConnection| { + let users = users::dsl::users + .filter(users::address.eq(self.authority.to_string())) + .select(Users::as_select()) + .limit(1) + .load(connection)?; + + let authority_id = if users.len() == 0 { + diesel::insert_into(users::table) + .values(vec![Users { + address: self.authority.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(users::id) + .get_result(connection)? + } else { + users.first().unwrap().id + }; -#[derive(Debug)] -pub struct AddBankEvent { - pub bank: Pubkey, - pub mint: Pubkey, - pub config: BankConfig, + let accounts = accounts::dsl::accounts + .filter(accounts::address.eq(self.account.to_string())) + .select(Accounts::as_select()) + .limit(1) + .load(connection)?; + + let account_id = if accounts.len() == 0 { + diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(accounts::id) + .get_result(connection)? + } else { + accounts.first().unwrap().id + }; + + let banks = banks::dsl::banks + .filter(banks::address.eq(self.bank.to_string())) + .select(Banks::as_select()) + .limit(1) + .load(connection)?; + + let bank_id = if banks.len() == 0 { + diesel::insert_into(banks::table) + .values(vec![Banks { + address: self.bank.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(banks::id) + .get_result(connection)? + } else { + banks.first().unwrap().id + }; + + let emissions_mints = mints::dsl::mints + .filter(mints::address.eq(self.emissions_mint.to_string())) + .select(Mints::as_select()) + .limit(1) + .load(connection)?; + + let emission_mint_id = if emissions_mints.len() == 0 { + diesel::insert_into(mints::table) + .values(vec![Mints { + address: self.emissions_mint.to_string(), + + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(mints::id) + .get_result(connection)? + } else { + emissions_mints.first().unwrap().id + }; + + let withdraw_emissions_event = WithdrawEmissionsEvents { + timestamp, + authority_id, + tx_sig, + call_stack, + in_flashloan, + account_id, + bank_id, + emission_mint_id, + amount: Decimal::from_u64(self.amount).unwrap(), + ..Default::default() + }; + + diesel::insert_into(withdraw_emissions_events::table) + .values(&withdraw_emissions_event) + .execute(connection)?; + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } } +// #[derive(Debug)] +// pub struct LiquidateEvent { +// pub asset_amount: u64, +// pub asset_bank: Pubkey, +// pub liability_bank: Pubkey, +// pub liquidator_account: Pubkey, +// pub liquidator_authority: Pubkey, +// pub liquidatee_account: Pubkey, +// } + +// impl MarginfiEvent for LiquidateEvent { +// fn db_insert( +// &self, +// timestamp: NaiveDateTime, +// tx_sig: String, +// in_flashloan: bool, +// call_stack: String, +// db_connection: &mut PgConnection, +// ) -> Result<(), IndexingError> { +// db_connection.transaction(|connection: &mut PgConnection| { +// let users = users::dsl::users +// .filter(users::address.eq(self.liquidator_authority.to_string())) +// .select(Users::as_select()) +// .limit(1) +// .load(connection)?; + +// let liquidator_authority_id = if users.len() == 0 { +// diesel::insert_into(users::table) +// .values(vec![Users { +// address: self.liquidator_authority.to_string(), +// ..Default::default() +// }]) +// .on_conflict_do_nothing() +// .returning(users::id) +// .get_result(connection)? +// } else { +// users.first().unwrap().id +// }; + +// let users = users::dsl::users +// .filter(users::address.eq(self.liquidator_account.to_string())) +// .select(Users::as_select()) +// .limit(1) +// .load(connection)?; + +// let liquidator_account_id = if users.len() == 0 { +// diesel::insert_into(users::table) +// .values(vec![Users { +// address: self.liquidator_account.to_string(), +// ..Default::default() +// }]) +// .on_conflict_do_nothing() +// .returning(users::id) +// .get_result(connection)? +// } else { +// users.first().unwrap().id +// }; + +// let users = users::dsl::users +// .filter(users::address.eq(self.liquidaee_account.to_string())) +// .select(Users::as_select()) +// .limit(1) +// .load(connection)?; + +// let liquidatee_account_id = if users.len() == 0 { +// diesel::insert_into(users::table) +// .values(vec![Users { +// address: self.liquidaee_account.to_string(), +// ..Default::default() +// }]) +// .on_conflict_do_nothing() +// .returning(users::id) +// .get_result(connection)? +// } else { +// users.first().unwrap().id +// }; + +// let banks = banks::dsl::banks +// .filter(banks::address.eq(self.asset_bank.to_string())) +// .select(Banks::as_select()) +// .limit(1) +// .load(connection)?; + +// let asset_bank_id = if banks.len() == 0 { +// diesel::insert_into(banks::table) +// .values(vec![Banks { +// address: self.asset_bank.to_string(), +// ..Default::default() +// }]) +// .on_conflict_do_nothing() +// .returning(banks::id) +// .get_result(connection)? +// } else { +// banks.first().unwrap().id +// }; + +// let banks = banks::dsl::banks +// .filter(banks::address.eq(self.liability_bank.to_string())) +// .select(Banks::as_select()) +// .limit(1) +// .load(connection)?; + +// let liability_bank_id = if banks.len() == 0 { +// diesel::insert_into(banks::table) +// .values(vec![Banks { +// address: self.liability_bank.to_string(), +// ..Default::default() +// }]) +// .on_conflict_do_nothing() +// .returning(banks::id) +// .get_result(connection)? +// } else { +// banks.first().unwrap().id +// }; + +// let liquidate_event = LiquidateEvents { +// timestamp, +// liquidator_authority_id, +// tx_sig, +// call_stack, +// in_flashloan, +// asset_bank_id, +// liability_bank_id, +// liquidator_account_id, +// liquidatee_account_id, +// asset_amount: Decimal::from_u64(self.asset_amount).unwrap(), +// ..Default::default() +// }; + +// diesel::insert_into(liquidate_events::table) +// .values(&liquidate_event) +// .execute(connection)?; + +// diesel::result::QueryResult::Ok(()) +// }) +// } +// } + +// #[derive(Debug)] +// pub struct AddBankEvent { +// pub bank: Pubkey, +// pub mint: Pubkey, +// pub config: BankConfig, +// } + +// impl MarginfiEvent for AddBankEvent { +// fn db_insert( +// &self, +// timestamp: NaiveDateTime, +// tx_sig: String, +// in_flashloan: bool, +// call_stack: String, +// db_connection: &mut PgConnection, +// ) -> Result<(), IndexingError> { +// todo!("AddBankEvent::db_insert") +// } +// } + pub struct MarginfiEventParser { program_id: Pubkey, marginfi_group: Pubkey, @@ -440,30 +1275,30 @@ impl MarginfiEventParser { all: instruction.withdraw_all.unwrap_or(false), })) } - LendingAccountLiquidate::DISCRIMINATOR => { - let marginfi_group = *ix_accounts.get(0).unwrap(); - if !marginfi_group.eq(&self.marginfi_group) { - return None; - } - - let instruction = - LendingAccountLiquidate::deserialize(&mut instruction_data).ok()?; - - let asset_bank = *ix_accounts.get(1).unwrap(); - let liability_bank = *ix_accounts.get(2).unwrap(); - let liquidator_account = *ix_accounts.get(3).unwrap(); - let liquidator_authority = *ix_accounts.get(4).unwrap(); - let liquidatee_account = *ix_accounts.get(5).unwrap(); - - Some(Event::Liquidate(LiquidateEvent { - asset_amount: instruction.asset_amount, - asset_bank, - liability_bank, - liquidator_account, - liquidator_authority, - liquidatee_account, - })) - } + // LendingAccountLiquidate::DISCRIMINATOR => { + // let marginfi_group = *ix_accounts.get(0).unwrap(); + // if !marginfi_group.eq(&self.marginfi_group) { + // return None; + // } + + // let instruction = + // LendingAccountLiquidate::deserialize(&mut instruction_data).ok()?; + + // let asset_bank = *ix_accounts.get(1).unwrap(); + // let liability_bank = *ix_accounts.get(2).unwrap(); + // let liquidator_account = *ix_accounts.get(3).unwrap(); + // let liquidator_authority = *ix_accounts.get(4).unwrap(); + // let liquidatee_account = *ix_accounts.get(5).unwrap(); + + // Some(Event::Liquidate(LiquidateEvent { + // asset_amount: instruction.asset_amount, + // asset_bank, + // liability_bank, + // liquidator_account, + // liquidator_authority, + // liquidatee_account, + // })) + // } LendingAccountWithdrawEmissions::DISCRIMINATOR => { let marginfi_group = *ix_accounts.get(0).unwrap(); if !marginfi_group.eq(&self.marginfi_group) { @@ -490,29 +1325,29 @@ impl MarginfiEventParser { amount: spl_transfer_amount, })) } - LendingPoolAddBank::DISCRIMINATOR => { - let marginfi_group = *ix_accounts.get(0).unwrap(); - if !marginfi_group.eq(&self.marginfi_group) { - return None; - } - - let bank_config = if slot < COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT { - BankConfig::deserialize(&mut &instruction_data[..531]).unwrap() - } else { - BankConfigCompact::deserialize(&mut &instruction_data[..531]) - .unwrap() - .into() - }; - - let bank_mint = *ix_accounts.get(3).unwrap(); - let bank = *ix_accounts.get(4).unwrap(); - - Some(Event::AddBank(AddBankEvent { - bank, - mint: bank_mint, - config: bank_config, - })) - } + // LendingPoolAddBank::DISCRIMINATOR => { + // let marginfi_group = *ix_accounts.get(0).unwrap(); + // if !marginfi_group.eq(&self.marginfi_group) { + // return None; + // } + + // let bank_config = if slot < COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT { + // BankConfig::deserialize(&mut &instruction_data[..531]).unwrap() + // } else { + // BankConfigCompact::deserialize(&mut &instruction_data[..531]) + // .unwrap() + // .into() + // }; + + // let bank_mint = *ix_accounts.get(3).unwrap(); + // let bank = *ix_accounts.get(4).unwrap(); + + // Some(Event::AddBank(AddBankEvent { + // bank, + // mint: bank_mint, + // config: bank_config, + // })) + // } LendingAccountStartFlashloan::DISCRIMINATOR => { *in_flashloan = true; From 5980aae04bd986d2acea01ca65d23991265300d4 Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:30:51 +0800 Subject: [PATCH 04/20] feat: more insertion logic --- observability/Cargo.lock | 122 ++--- .../2024-03-14-151141_initial_tables/up.sql | 17 + .../crates/event_indexer/src/db/models.rs | 17 + .../crates/event_indexer/src/db/schema.rs | 20 + .../crates/event_indexer/src/entity_store.rs | 85 ++-- .../crates/event_indexer/src/error.rs | 13 +- .../crates/event_indexer/src/parser.rs | 454 +++++++++++------- 7 files changed, 458 insertions(+), 270 deletions(-) diff --git a/observability/Cargo.lock b/observability/Cargo.lock index 4b40bebf..7ff4ee93 100644 --- a/observability/Cargo.lock +++ b/observability/Cargo.lock @@ -89,9 +89,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -133,7 +133,7 @@ checksum = "f468970344c7c9f9d03b4da854fd7c54f21305059f53789d0045c1dd803f0018" dependencies = [ "anchor-syn", "anyhow", - "bs58 0.5.0", + "bs58 0.5.1", "proc-macro2 1.0.79", "quote 1.0.35", "rustversion", @@ -257,7 +257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a125e4b0cc046cfec58f5aa25038e34cf440151d58f0db3afc55308251fe936d" dependencies = [ "anyhow", - "bs58 0.5.0", + "bs58 0.5.1", "heck 0.3.3", "proc-macro2 1.0.79", "quote 1.0.35", @@ -537,18 +537,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "461abc97219de0eaaf81fe3ef974a540158f3d079c2ab200f891f1a2ef201e85" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -644,9 +644,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "95d8e92cac0961e91dbd517496b00f7e9b92363dbe6d42c3198268323798860c" dependencies = [ "addr2line", "cc", @@ -711,9 +711,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bitmaps" @@ -855,9 +855,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -882,9 +882,9 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bs58" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ "tinyvec", ] @@ -922,7 +922,7 @@ checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1254,7 +1254,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "strsim 0.10.0", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1265,7 +1265,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1341,7 +1341,7 @@ version = "2.1.4" source = "git+https://github.com/diesel-rs/diesel.git?rev=12429cb4eebfb50581c494e1a183e4faed65c47f#12429cb4eebfb50581c494e1a183e4faed65c47f" dependencies = [ "bigdecimal", - "bitflags 2.4.2", + "bitflags 2.5.0", "byteorder", "chrono", "diesel_derives", @@ -1361,7 +1361,7 @@ dependencies = [ "dsl_auto_type", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1369,7 +1369,7 @@ name = "diesel_table_macro_syntax" version = "0.1.0" source = "git+https://github.com/diesel-rs/diesel.git?rev=12429cb4eebfb50581c494e1a183e4faed65c47f#12429cb4eebfb50581c494e1a183e4faed65c47f" dependencies = [ - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1400,7 +1400,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1442,7 +1442,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1530,7 +1530,7 @@ checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1542,7 +1542,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1782,7 +1782,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -1881,9 +1881,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" dependencies = [ "bytes", "fnv", @@ -2609,7 +2609,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -2711,7 +2711,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -2723,7 +2723,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -2878,7 +2878,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -2962,7 +2962,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2 1.0.79", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -3062,7 +3062,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.52", + "syn 2.0.53", "tempfile", "which", ] @@ -3077,7 +3077,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -3373,9 +3373,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.26" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "async-compression", "base64 0.21.7", @@ -3519,11 +3519,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -3653,7 +3653,7 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -3721,7 +3721,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -3777,7 +3777,7 @@ dependencies = [ "darling", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -3888,9 +3888,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" @@ -4086,7 +4086,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "rustc_version", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -4462,7 +4462,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "rustversion", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -4744,7 +4744,7 @@ checksum = "07fd7858fc4ff8fb0e34090e41d7eb06a823e1057945c26d480bfc21d2338a93" dependencies = [ "quote 1.0.35", "spl-discriminator-syn", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -4756,7 +4756,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "sha2 0.10.8", - "syn 2.0.52", + "syn 2.0.53", "thiserror", ] @@ -4813,7 +4813,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "sha2 0.10.8", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -5011,9 +5011,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.52" +version = "2.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", @@ -5112,7 +5112,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -5227,7 +5227,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -5370,7 +5370,7 @@ dependencies = [ "proc-macro2 1.0.79", "prost-build", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -5437,7 +5437,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -5737,7 +5737,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", "wasm-bindgen-shared", ] @@ -5771,7 +5771,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6151,7 +6151,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] @@ -6171,7 +6171,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.52", + "syn 2.0.53", ] [[package]] diff --git a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql index cffe6236..64943bdf 100644 --- a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql +++ b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql @@ -140,3 +140,20 @@ CREATE TABLE "withdraw_emissions_events"( "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); SELECT diesel_manage_updated_at('withdraw_emissions_events'); + +CREATE TABLE "liquidate_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + "liquidator_account_id" INT4 NOT NULL REFERENCES "accounts"("id"), + "liquidatee_account_id" INT4 NOT NULL REFERENCES "accounts"("id"), + "liquidator_user_id" INT4 NOT NULL REFERENCES "users"("id"), + "asset_bank_id" INT4 NOT NULL REFERENCES "banks"("id"), + "liability_bank_id" INT4 NOT NULL REFERENCES "banks"("id"), + "asset_amount" NUMERIC NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() +); +SELECT diesel_manage_updated_at('liquidate_events'); diff --git a/observability/crates/event_indexer/src/db/models.rs b/observability/crates/event_indexer/src/db/models.rs index 00a8ad8e..0df75983 100644 --- a/observability/crates/event_indexer/src/db/models.rs +++ b/observability/crates/event_indexer/src/db/models.rs @@ -143,3 +143,20 @@ pub struct WithdrawEmissionsEvents { pub emission_mint_id: i32, pub amount: Decimal, } + +#[derive(Default, Debug, Queryable, Selectable, Insertable)] +#[diesel(table_name = liquidate_events)] +pub struct LiquidateEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + pub liquidator_account_id: i32, + pub liquidatee_account_id: i32, + pub liquidator_user_id: i32, + pub asset_bank_id: i32, + pub liability_bank_id: i32, + pub asset_amount: Decimal, +} diff --git a/observability/crates/event_indexer/src/db/schema.rs b/observability/crates/event_indexer/src/db/schema.rs index 5b7160fe..45c30186 100644 --- a/observability/crates/event_indexer/src/db/schema.rs +++ b/observability/crates/event_indexer/src/db/schema.rs @@ -66,6 +66,24 @@ diesel::table! { } } +diesel::table! { + liquidate_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + liquidator_account_id -> Int4, + liquidatee_account_id -> Int4, + liquidator_user_id -> Int4, + asset_bank_id -> Int4, + liability_bank_id -> Int4, + asset_amount -> Numeric, + created_at -> Timestamp, + updated_at -> Timestamp, + } +} + diesel::table! { mints (id) { id -> Int4, @@ -162,6 +180,7 @@ diesel::joinable!(create_account_events -> users (authority_id)); diesel::joinable!(deposit_events -> accounts (account_id)); diesel::joinable!(deposit_events -> banks (bank_id)); diesel::joinable!(deposit_events -> users (authority_id)); +diesel::joinable!(liquidate_events -> users (liquidator_user_id)); diesel::joinable!(repay_events -> accounts (account_id)); diesel::joinable!(repay_events -> banks (bank_id)); diesel::joinable!(repay_events -> users (authority_id)); @@ -180,6 +199,7 @@ diesel::allow_tables_to_appear_in_same_query!( borrow_events, create_account_events, deposit_events, + liquidate_events, mints, repay_events, transfer_account_authority_events, diff --git a/observability/crates/event_indexer/src/entity_store.rs b/observability/crates/event_indexer/src/entity_store.rs index 9373403c..d6a29ced 100644 --- a/observability/crates/event_indexer/src/entity_store.rs +++ b/observability/crates/event_indexer/src/entity_store.rs @@ -13,7 +13,7 @@ use spl_token::state::Mint; use crate::{ db::{establish_connection, models::*, schema::*}, - error::IndexingError, + error::{FetchEntityError, IndexingError}, }; pub struct EntityStore { @@ -90,14 +90,8 @@ impl MintData { } let mint = MintData::fetch_from_rpc(rpc_client, address)?; - if let Some(mint) = mint { - Ok(mint) - } else { - Err(IndexingError::FailedToFetchEntity(format!( - "Mint not found: {}", - address - ))) - } + + Ok(mint) } fn fetch_from_db( @@ -109,7 +103,12 @@ impl MintData { .select(Mints::as_select()) .limit(1) .load(db_connection) - .map_err(|e| IndexingError::FailedToFetchEntity(e.to_string()))?; + .map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( + "mint".to_string(), + e.to_string(), + )) + })?; if db_records.is_empty() { return Ok(None); @@ -125,24 +124,29 @@ impl MintData { })) } - fn fetch_from_rpc( - rpc_client: &RpcClient, - address: &str, - ) -> Result, IndexingError> { + fn fetch_from_rpc(rpc_client: &RpcClient, address: &str) -> Result { let mint_data = rpc_client .get_account_data(&Pubkey::from_str(address).unwrap()) - .map_err(|e| IndexingError::FailedToFetchEntity(e.to_string()))?; + .map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( + "mint".to_string(), + e.to_string(), + )) + })?; let mint = Mint::unpack_from_slice(&mint_data).map_err(|e| { - IndexingError::FailedToFetchEntity(format!("Failed to unpack mint: {}", e)) + IndexingError::FailedToFetchEntity(FetchEntityError::UnpackError( + "mint".to_string(), + e.to_string(), + )) })?; - Ok(Some(MintData { + Ok(MintData { id: None, address: address.to_string(), symbol: "".to_string(), decimals: mint.decimals as i16, - })) + }) } } @@ -166,14 +170,8 @@ impl BankData { } let mint = BankData::fetch_from_rpc(rpc_client, db_connection, address)?; - if let Some(mint) = mint { - Ok(mint) - } else { - Err(IndexingError::FailedToFetchEntity(format!( - "Bank not found: {}", - address - ))) - } + + Ok(mint) } fn fetch_from_db( @@ -185,7 +183,12 @@ impl BankData { .select(Banks::as_select()) .limit(1) .load(db_connection) - .map_err(|e| IndexingError::FailedToFetchEntity(e.to_string()))?; + .map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( + "bank".to_string(), + e.to_string(), + )) + })?; if db_records.is_empty() { return Ok(None); @@ -193,13 +196,9 @@ impl BankData { let db_record = db_records.get(0).unwrap(); - let mint_data = MintData::fetch_from_db(db_connection, &db_record.mint_id.to_string())? - .ok_or_else(|| { - IndexingError::FailedToFetchEntity(format!( - "Mint not found in DB: {}", - db_record.mint_id - )) - })?; + let mint_data = MintData::fetch_from_db(db_connection, &db_record.mint_id.to_string()) + .unwrap() + .unwrap(); Ok(Some(BankData { id: Some(db_record.id), @@ -212,22 +211,30 @@ impl BankData { rpc_client: &RpcClient, db_connection: &mut PgConnection, address: &str, - ) -> Result, IndexingError> { + ) -> Result { let mint_data = rpc_client .get_account_data(&Pubkey::from_str(address).unwrap()) - .map_err(|e| IndexingError::FailedToFetchEntity(e.to_string()))?; + .map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( + "bank".to_string(), + e.to_string(), + )) + })?; let bank = Bank::try_deserialize(&mut mint_data.as_slice()).map_err(|e| { - IndexingError::FailedToFetchEntity(format!("Failed to unpack bank: {}", e)) + IndexingError::FailedToFetchEntity(FetchEntityError::UnpackError( + "bank".to_string(), + e.to_string(), + )) })?; let mint_data = MintData::fetch(rpc_client, db_connection, &bank.mint.to_string())?; - Ok(Some(BankData { + Ok(BankData { id: None, address: address.to_string(), mint: mint_data, - })) + }) } } diff --git a/observability/crates/event_indexer/src/error.rs b/observability/crates/event_indexer/src/error.rs index 7c8b1100..2d677c1e 100644 --- a/observability/crates/event_indexer/src/error.rs +++ b/observability/crates/event_indexer/src/error.rs @@ -15,8 +15,8 @@ pub enum IndexingError { #[error("Failed to find slot for tx signature {0:?}")] FailedToFindTransactionSlot(Signature), - #[error("Failed to fetch entity: {0}")] - FailedToFetchEntity(String), + #[error(transparent)] + FailedToFetchEntity(FetchEntityError), #[error("Failed to insert event: {0}")] FailedToInsertEvent(String), @@ -24,3 +24,12 @@ pub enum IndexingError { #[error("An unknown error occurred")] Unknown, } + +#[derive(Error, Debug)] +pub enum FetchEntityError { + #[error("Failed to fetch {0} entity: {1}")] + FetchError(String, String), + + #[error("Failed to unpack {0} entity: {1}")] + UnpackError(String, String), +} diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 1fdb303a..61760eef 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -8,10 +8,10 @@ use diesel::{ use enum_dispatch::enum_dispatch; use marginfi::instruction::{ LendingAccountBorrow, LendingAccountCloseBalance, LendingAccountDeposit, - LendingAccountEndFlashloan, LendingAccountRepay, LendingAccountSettleEmissions, - LendingAccountStartFlashloan, LendingAccountWithdraw, LendingAccountWithdrawEmissions, - LendingPoolAccrueBankInterest, LendingPoolAddBankWithSeed, LendingPoolConfigureBank, - MarginfiAccountInitialize, SetNewAccountAuthority, + LendingAccountEndFlashloan, LendingAccountLiquidate, LendingAccountRepay, + LendingAccountSettleEmissions, LendingAccountStartFlashloan, LendingAccountWithdraw, + LendingAccountWithdrawEmissions, LendingPoolAccrueBankInterest, LendingPoolAddBankWithSeed, + LendingPoolConfigureBank, MarginfiAccountInitialize, SetNewAccountAuthority, }; use rust_decimal::{prelude::FromPrimitive, Decimal}; use solana_sdk::{ @@ -71,8 +71,7 @@ pub enum Event { Repay(RepayEvent), Withdraw(WithdrawEvent), WithdrawEmissions(WithdrawEmissionsEvent), - // Liquidate(LiquidateEvent), - + Liquidate(LiquidateEvent), // // Admin actions // AddBank(AddBankEvent), } @@ -91,7 +90,7 @@ impl MarginfiEvent for CreateAccountEvent { in_flashloan: bool, call_stack: String, db_connection: &mut PgConnection, - entity_store: &mut EntityStore, + _entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { db_connection .transaction(|connection: &mut PgConnection| { @@ -169,7 +168,7 @@ impl MarginfiEvent for AccountAuthorityTransferEvent { in_flashloan: bool, call_stack: String, db_connection: &mut PgConnection, - entity_store: &mut EntityStore, + _entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { db_connection .transaction(|connection: &mut PgConnection| { @@ -396,6 +395,10 @@ impl MarginfiEvent for BorrowEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let bank_data = entity_store + .get_or_fetch_bank(&self.bank.to_string()) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + db_connection .transaction(|connection: &mut PgConnection| { let users = users::dsl::users @@ -437,6 +440,27 @@ impl MarginfiEvent for BorrowEvent { accounts.first().unwrap().id }; + let mints = mints::dsl::mints + .filter(mints::address.eq(bank_data.mint.address.clone())) + .select(Mints::as_select()) + .limit(1) + .load(connection)?; + + let mint_id = if mints.len() == 0 { + diesel::insert_into(mints::table) + .values(vec![Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(mints::id) + .get_result(connection)? + } else { + mints.first().unwrap().id + }; + let banks = banks::dsl::banks .filter(banks::address.eq(self.bank.to_string())) .select(Banks::as_select()) @@ -447,6 +471,7 @@ impl MarginfiEvent for BorrowEvent { diesel::insert_into(banks::table) .values(vec![Banks { address: self.bank.to_string(), + mint_id, ..Default::default() }]) .on_conflict_do_nothing() @@ -497,6 +522,10 @@ impl MarginfiEvent for RepayEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let bank_data = entity_store + .get_or_fetch_bank(&self.bank.to_string()) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + db_connection .transaction(|connection: &mut PgConnection| { let users = users::dsl::users @@ -538,6 +567,27 @@ impl MarginfiEvent for RepayEvent { accounts.first().unwrap().id }; + let mints = mints::dsl::mints + .filter(mints::address.eq(bank_data.mint.address.clone())) + .select(Mints::as_select()) + .limit(1) + .load(connection)?; + + let mint_id = if mints.len() == 0 { + diesel::insert_into(mints::table) + .values(vec![Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(mints::id) + .get_result(connection)? + } else { + mints.first().unwrap().id + }; + let banks = banks::dsl::banks .filter(banks::address.eq(self.bank.to_string())) .select(Banks::as_select()) @@ -548,6 +598,7 @@ impl MarginfiEvent for RepayEvent { diesel::insert_into(banks::table) .values(vec![Banks { address: self.bank.to_string(), + mint_id, ..Default::default() }]) .on_conflict_do_nothing() @@ -599,6 +650,10 @@ impl MarginfiEvent for WithdrawEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let bank_data = entity_store + .get_or_fetch_bank(&self.bank.to_string()) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + db_connection .transaction(|connection: &mut PgConnection| { let users = users::dsl::users @@ -640,6 +695,27 @@ impl MarginfiEvent for WithdrawEvent { accounts.first().unwrap().id }; + let mints = mints::dsl::mints + .filter(mints::address.eq(bank_data.mint.address.clone())) + .select(Mints::as_select()) + .limit(1) + .load(connection)?; + + let mint_id = if mints.len() == 0 { + diesel::insert_into(mints::table) + .values(vec![Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(mints::id) + .get_result(connection)? + } else { + mints.first().unwrap().id + }; + let banks = banks::dsl::banks .filter(banks::address.eq(self.bank.to_string())) .select(Banks::as_select()) @@ -650,6 +726,7 @@ impl MarginfiEvent for WithdrawEvent { diesel::insert_into(banks::table) .values(vec![Banks { address: self.bank.to_string(), + mint_id, ..Default::default() }]) .on_conflict_do_nothing() @@ -701,6 +778,13 @@ impl MarginfiEvent for WithdrawEmissionsEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let bank_data = entity_store + .get_or_fetch_bank(&self.bank.to_string()) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + + let emission_mint_data = + entity_store.get_or_fetch_mint(&self.emissions_mint.to_string())?; + db_connection .transaction(|connection: &mut PgConnection| { let users = users::dsl::users @@ -742,6 +826,27 @@ impl MarginfiEvent for WithdrawEmissionsEvent { accounts.first().unwrap().id }; + let mints = mints::dsl::mints + .filter(mints::address.eq(bank_data.mint.address.clone())) + .select(Mints::as_select()) + .limit(1) + .load(connection)?; + + let mint_id = if mints.len() == 0 { + diesel::insert_into(mints::table) + .values(vec![Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(mints::id) + .get_result(connection)? + } else { + mints.first().unwrap().id + }; + let banks = banks::dsl::banks .filter(banks::address.eq(self.bank.to_string())) .select(Banks::as_select()) @@ -752,6 +857,7 @@ impl MarginfiEvent for WithdrawEmissionsEvent { diesel::insert_into(banks::table) .values(vec![Banks { address: self.bank.to_string(), + mint_id, ..Default::default() }]) .on_conflict_do_nothing() @@ -771,7 +877,8 @@ impl MarginfiEvent for WithdrawEmissionsEvent { diesel::insert_into(mints::table) .values(vec![Mints { address: self.emissions_mint.to_string(), - + symbol: emission_mint_data.symbol.clone(), + decimals: emission_mint_data.decimals, ..Default::default() }]) .on_conflict_do_nothing() @@ -804,143 +911,154 @@ impl MarginfiEvent for WithdrawEmissionsEvent { } } -// #[derive(Debug)] -// pub struct LiquidateEvent { -// pub asset_amount: u64, -// pub asset_bank: Pubkey, -// pub liability_bank: Pubkey, -// pub liquidator_account: Pubkey, -// pub liquidator_authority: Pubkey, -// pub liquidatee_account: Pubkey, -// } +#[derive(Debug)] +pub struct LiquidateEvent { + pub asset_amount: u64, + pub asset_bank: Pubkey, + pub liability_bank: Pubkey, + pub liquidator_account: Pubkey, + pub liquidator_authority: Pubkey, + pub liquidatee_account: Pubkey, +} -// impl MarginfiEvent for LiquidateEvent { -// fn db_insert( -// &self, -// timestamp: NaiveDateTime, -// tx_sig: String, -// in_flashloan: bool, -// call_stack: String, -// db_connection: &mut PgConnection, -// ) -> Result<(), IndexingError> { -// db_connection.transaction(|connection: &mut PgConnection| { -// let users = users::dsl::users -// .filter(users::address.eq(self.liquidator_authority.to_string())) -// .select(Users::as_select()) -// .limit(1) -// .load(connection)?; - -// let liquidator_authority_id = if users.len() == 0 { -// diesel::insert_into(users::table) -// .values(vec![Users { -// address: self.liquidator_authority.to_string(), -// ..Default::default() -// }]) -// .on_conflict_do_nothing() -// .returning(users::id) -// .get_result(connection)? -// } else { -// users.first().unwrap().id -// }; - -// let users = users::dsl::users -// .filter(users::address.eq(self.liquidator_account.to_string())) -// .select(Users::as_select()) -// .limit(1) -// .load(connection)?; - -// let liquidator_account_id = if users.len() == 0 { -// diesel::insert_into(users::table) -// .values(vec![Users { -// address: self.liquidator_account.to_string(), -// ..Default::default() -// }]) -// .on_conflict_do_nothing() -// .returning(users::id) -// .get_result(connection)? -// } else { -// users.first().unwrap().id -// }; - -// let users = users::dsl::users -// .filter(users::address.eq(self.liquidaee_account.to_string())) -// .select(Users::as_select()) -// .limit(1) -// .load(connection)?; - -// let liquidatee_account_id = if users.len() == 0 { -// diesel::insert_into(users::table) -// .values(vec![Users { -// address: self.liquidaee_account.to_string(), -// ..Default::default() -// }]) -// .on_conflict_do_nothing() -// .returning(users::id) -// .get_result(connection)? -// } else { -// users.first().unwrap().id -// }; - -// let banks = banks::dsl::banks -// .filter(banks::address.eq(self.asset_bank.to_string())) -// .select(Banks::as_select()) -// .limit(1) -// .load(connection)?; - -// let asset_bank_id = if banks.len() == 0 { -// diesel::insert_into(banks::table) -// .values(vec![Banks { -// address: self.asset_bank.to_string(), -// ..Default::default() -// }]) -// .on_conflict_do_nothing() -// .returning(banks::id) -// .get_result(connection)? -// } else { -// banks.first().unwrap().id -// }; - -// let banks = banks::dsl::banks -// .filter(banks::address.eq(self.liability_bank.to_string())) -// .select(Banks::as_select()) -// .limit(1) -// .load(connection)?; - -// let liability_bank_id = if banks.len() == 0 { -// diesel::insert_into(banks::table) -// .values(vec![Banks { -// address: self.liability_bank.to_string(), -// ..Default::default() -// }]) -// .on_conflict_do_nothing() -// .returning(banks::id) -// .get_result(connection)? -// } else { -// banks.first().unwrap().id -// }; - -// let liquidate_event = LiquidateEvents { -// timestamp, -// liquidator_authority_id, -// tx_sig, -// call_stack, -// in_flashloan, -// asset_bank_id, -// liability_bank_id, -// liquidator_account_id, -// liquidatee_account_id, -// asset_amount: Decimal::from_u64(self.asset_amount).unwrap(), -// ..Default::default() -// }; - -// diesel::insert_into(liquidate_events::table) -// .values(&liquidate_event) -// .execute(connection)?; - -// diesel::result::QueryResult::Ok(()) -// }) -// } -// } +impl MarginfiEvent for LiquidateEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + let asset_bank_data = entity_store + .get_or_fetch_bank(&self.asset_bank.to_string()) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + + let liability_bank_data = entity_store + .get_or_fetch_bank(&self.liability_bank.to_string()) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + + db_connection + .transaction(|connection: &mut PgConnection| { + let users = users::dsl::users + .filter(users::address.eq(self.liquidator_authority.to_string())) + .select(Users::as_select()) + .limit(1) + .load(connection)?; + + let liquidator_user_id = if users.len() == 0 { + diesel::insert_into(users::table) + .values(vec![Users { + address: self.liquidator_authority.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(users::id) + .get_result(connection)? + } else { + users.first().unwrap().id + }; + + let accounts = accounts::dsl::accounts + .filter(accounts::address.eq(self.liquidator_account.to_string())) + .select(Accounts::as_select()) + .limit(1) + .load(connection)?; + + let liquidator_account_id = if accounts.len() == 0 { + diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: self.liquidator_account.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(accounts::id) + .get_result(connection)? + } else { + accounts.first().unwrap().id + }; + + let accounts = accounts::dsl::accounts + .filter(accounts::address.eq(self.liquidatee_account.to_string())) + .select(Accounts::as_select()) + .limit(1) + .load(connection)?; + + let liquidatee_account_id = if accounts.len() == 0 { + diesel::insert_into(accounts::table) + .values(vec![Accounts { + address: self.liquidatee_account.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(accounts::id) + .get_result(connection)? + } else { + accounts.first().unwrap().id + }; + + let banks = banks::dsl::banks + .filter(banks::address.eq(self.asset_bank.to_string())) + .select(Banks::as_select()) + .limit(1) + .load(connection)?; + + let asset_bank_id = if banks.len() == 0 { + diesel::insert_into(banks::table) + .values(vec![Banks { + address: self.asset_bank.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(banks::id) + .get_result(connection)? + } else { + banks.first().unwrap().id + }; + + let banks = banks::dsl::banks + .filter(banks::address.eq(self.liability_bank.to_string())) + .select(Banks::as_select()) + .limit(1) + .load(connection)?; + + let liability_bank_id = if banks.len() == 0 { + diesel::insert_into(banks::table) + .values(vec![Banks { + address: self.liability_bank.to_string(), + ..Default::default() + }]) + .on_conflict_do_nothing() + .returning(banks::id) + .get_result(connection)? + } else { + banks.first().unwrap().id + }; + + let liquidate_event = LiquidateEvents { + timestamp, + tx_sig, + call_stack, + in_flashloan, + liquidator_account_id, + liquidatee_account_id, + liquidator_user_id, + asset_bank_id, + liability_bank_id, + asset_amount: Decimal::from_u64(self.asset_amount).unwrap(), + ..Default::default() + }; + + diesel::insert_into(liquidate_events::table) + .values(&liquidate_event) + .execute(connection)?; + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } +} // #[derive(Debug)] // pub struct AddBankEvent { @@ -1275,30 +1393,30 @@ impl MarginfiEventParser { all: instruction.withdraw_all.unwrap_or(false), })) } - // LendingAccountLiquidate::DISCRIMINATOR => { - // let marginfi_group = *ix_accounts.get(0).unwrap(); - // if !marginfi_group.eq(&self.marginfi_group) { - // return None; - // } + LendingAccountLiquidate::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } - // let instruction = - // LendingAccountLiquidate::deserialize(&mut instruction_data).ok()?; - - // let asset_bank = *ix_accounts.get(1).unwrap(); - // let liability_bank = *ix_accounts.get(2).unwrap(); - // let liquidator_account = *ix_accounts.get(3).unwrap(); - // let liquidator_authority = *ix_accounts.get(4).unwrap(); - // let liquidatee_account = *ix_accounts.get(5).unwrap(); - - // Some(Event::Liquidate(LiquidateEvent { - // asset_amount: instruction.asset_amount, - // asset_bank, - // liability_bank, - // liquidator_account, - // liquidator_authority, - // liquidatee_account, - // })) - // } + let instruction = + LendingAccountLiquidate::deserialize(&mut instruction_data).ok()?; + + let asset_bank = *ix_accounts.get(1).unwrap(); + let liability_bank = *ix_accounts.get(2).unwrap(); + let liquidator_account = *ix_accounts.get(3).unwrap(); + let liquidator_authority = *ix_accounts.get(4).unwrap(); + let liquidatee_account = *ix_accounts.get(5).unwrap(); + + Some(Event::Liquidate(LiquidateEvent { + asset_amount: instruction.asset_amount, + asset_bank, + liability_bank, + liquidator_account, + liquidator_authority, + liquidatee_account, + })) + } LendingAccountWithdrawEmissions::DISCRIMINATOR => { let marginfi_group = *ix_accounts.get(0).unwrap(); if !marginfi_group.eq(&self.marginfi_group) { From ca8051b9b149c2b728d762955fee9c358140a03c Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Tue, 26 Mar 2024 22:33:48 +0800 Subject: [PATCH 05/20] feat: more insertion logic --- .../crates/event_indexer/src/db/models.rs | 5 +- .../crates/event_indexer/src/entity_store.rs | 127 ++- observability/crates/event_indexer/src/lib.rs | 1 + .../crates/event_indexer/src/macros.rs | 22 + .../crates/event_indexer/src/parser.rs | 990 +++++++----------- 5 files changed, 521 insertions(+), 624 deletions(-) create mode 100644 observability/crates/event_indexer/src/macros.rs diff --git a/observability/crates/event_indexer/src/db/models.rs b/observability/crates/event_indexer/src/db/models.rs index 0df75983..79716039 100644 --- a/observability/crates/event_indexer/src/db/models.rs +++ b/observability/crates/event_indexer/src/db/models.rs @@ -29,8 +29,9 @@ pub struct Users { pub address: String, } -#[derive(Default, Debug, Queryable, Selectable, Insertable)] +#[derive(Default, Debug, Queryable, Selectable, Insertable, Associations)] #[diesel(table_name = accounts)] +#[diesel(belongs_to(Users, foreign_key = user_id))] pub struct Accounts { #[diesel(skip_insertion)] pub id: i32, @@ -39,8 +40,8 @@ pub struct Accounts { } #[derive(Default, Debug, Queryable, Selectable, Insertable, Associations)] -#[diesel(belongs_to(Accounts, foreign_key = account_id), belongs_to(Users, foreign_key = authority_id))] #[diesel(table_name = create_account_events)] +#[diesel(belongs_to(Accounts, foreign_key = account_id), belongs_to(Users, foreign_key = authority_id))] pub struct CreateAccountEvents { #[diesel(skip_insertion)] pub id: i32, diff --git a/observability/crates/event_indexer/src/entity_store.rs b/observability/crates/event_indexer/src/entity_store.rs index d6a29ced..ebecbbc6 100644 --- a/observability/crates/event_indexer/src/entity_store.rs +++ b/observability/crates/event_indexer/src/entity_store.rs @@ -1,8 +1,10 @@ use std::{collections::HashMap, str::FromStr}; use anchor_lang::AccountDeserialize; -use diesel::{ExpressionMethods, PgConnection, QueryDsl, RunQueryDsl, SelectableHelper}; -use marginfi::state::marginfi_group::Bank; +use diesel::{ + prelude::*, ExpressionMethods, PgConnection, QueryDsl, RunQueryDsl, SelectableHelper, +}; +use marginfi::state::{marginfi_account::MarginfiAccount, marginfi_group::Bank}; use solana_client::rpc_client::RpcClient; use solana_sdk::{ commitment_config::{CommitmentConfig, CommitmentLevel}, @@ -67,6 +69,17 @@ impl EntityStore { Ok(bank) } } + + pub fn get_or_fetch_account(&mut self, address: &str) -> Result { + let maybe_account = self.account_cache.get(&address.to_string()); + + if let Some(account) = maybe_account { + Ok(account.clone()) + } else { + let account = AccountData::fetch(&self.rpc_client, &mut self.db_connection, address)?; + Ok(account) + } + } } #[derive(Debug, Clone)] @@ -82,14 +95,14 @@ impl MintData { rpc_client: &RpcClient, db_connection: &mut PgConnection, address: &str, - ) -> Result { - let db_record = MintData::fetch_from_db(db_connection, address)?; + ) -> Result { + let db_record = Self::fetch_from_db(db_connection, address)?; if let Some(db_record) = db_record { return Ok(db_record); } - let mint = MintData::fetch_from_rpc(rpc_client, address)?; + let mint = Self::fetch_from_rpc(rpc_client, address)?; Ok(mint) } @@ -97,7 +110,7 @@ impl MintData { fn fetch_from_db( db_connection: &mut PgConnection, address: &str, - ) -> Result, IndexingError> { + ) -> Result, IndexingError> { let db_records = mints::dsl::mints .filter(mints::address.eq(address.to_string())) .select(Mints::as_select()) @@ -116,7 +129,7 @@ impl MintData { let db_record = db_records.get(0).unwrap(); - Ok(Some(MintData { + Ok(Some(Self { id: Some(db_record.id), address: db_record.address.clone(), symbol: db_record.symbol.clone(), @@ -124,7 +137,7 @@ impl MintData { })) } - fn fetch_from_rpc(rpc_client: &RpcClient, address: &str) -> Result { + fn fetch_from_rpc(rpc_client: &RpcClient, address: &str) -> Result { let mint_data = rpc_client .get_account_data(&Pubkey::from_str(address).unwrap()) .map_err(|e| { @@ -141,7 +154,7 @@ impl MintData { )) })?; - Ok(MintData { + Ok(Self { id: None, address: address.to_string(), symbol: "".to_string(), @@ -162,14 +175,14 @@ impl BankData { rpc_client: &RpcClient, db_connection: &mut PgConnection, address: &str, - ) -> Result { - let db_record = BankData::fetch_from_db(db_connection, address)?; + ) -> Result { + let db_record = Self::fetch_from_db(db_connection, address)?; if let Some(db_record) = db_record { return Ok(db_record); } - let mint = BankData::fetch_from_rpc(rpc_client, db_connection, address)?; + let mint = Self::fetch_from_rpc(rpc_client, db_connection, address)?; Ok(mint) } @@ -177,7 +190,7 @@ impl BankData { fn fetch_from_db( db_connection: &mut PgConnection, address: &str, - ) -> Result, IndexingError> { + ) -> Result, IndexingError> { let db_records = banks::dsl::banks .filter(banks::address.eq(address.to_string())) .select(Banks::as_select()) @@ -200,7 +213,7 @@ impl BankData { .unwrap() .unwrap(); - Ok(Some(BankData { + Ok(Some(Self { id: Some(db_record.id), address: db_record.address.clone(), mint: mint_data, @@ -211,7 +224,7 @@ impl BankData { rpc_client: &RpcClient, db_connection: &mut PgConnection, address: &str, - ) -> Result { + ) -> Result { let mint_data = rpc_client .get_account_data(&Pubkey::from_str(address).unwrap()) .map_err(|e| { @@ -230,7 +243,7 @@ impl BankData { let mint_data = MintData::fetch(rpc_client, db_connection, &bank.mint.to_string())?; - Ok(BankData { + Ok(Self { id: None, address: address.to_string(), mint: mint_data, @@ -242,12 +255,88 @@ impl BankData { pub struct AccountData { pub id: Option, pub address: String, - pub user_id: i32, + pub authority: String, } impl AccountData { - fn get_cache_or_fetch(&mut self, address: &str) -> AccountData { - todo!() + pub fn fetch( + rpc_client: &RpcClient, + db_connection: &mut PgConnection, + address: &str, + ) -> Result { + let db_record = Self::fetch_from_db(db_connection, address)?; + + if let Some(db_record) = db_record { + return Ok(db_record); + } + + let mint = Self::fetch_from_rpc(rpc_client, address)?; + + Ok(mint) + } + + fn fetch_from_db( + db_connection: &mut PgConnection, + address: &str, + ) -> Result, IndexingError> { + let maybe_db_record = accounts::dsl::accounts + .filter(accounts::address.eq(address.to_string())) + .select(Accounts::as_select()) + .get_result(db_connection) + .optional() + .map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( + "account".to_string(), + e.to_string(), + )) + })?; + + if maybe_db_record.is_none() { + return Ok(None); + } + + let db_record = maybe_db_record.unwrap(); + + let authority = users::dsl::users + .find(&db_record.user_id) + .select(Users::as_select()) + .first(db_connection) + .map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( + "account".to_string(), + e.to_string(), + )) + })?; + + Ok(Some(Self { + id: Some(db_record.id), + address: db_record.address.clone(), + authority: authority.address.clone(), + })) + } + + fn fetch_from_rpc(rpc_client: &RpcClient, address: &str) -> Result { + let mint_data = rpc_client + .get_account_data(&Pubkey::from_str(address).unwrap()) + .map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( + "account".to_string(), + e.to_string(), + )) + })?; + + let account = MarginfiAccount::try_deserialize(&mut mint_data.as_slice()).map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::UnpackError( + "account".to_string(), + e.to_string(), + )) + })?; + + Ok(Self { + id: None, + address: address.to_string(), + authority: account.authority.to_string(), + }) } } diff --git a/observability/crates/event_indexer/src/lib.rs b/observability/crates/event_indexer/src/lib.rs index 1c9fc704..bc7541de 100644 --- a/observability/crates/event_indexer/src/lib.rs +++ b/observability/crates/event_indexer/src/lib.rs @@ -3,5 +3,6 @@ pub mod db; pub mod entity_store; pub mod error; pub mod indexer; +pub mod macros; pub mod parser; pub mod snapshot; diff --git a/observability/crates/event_indexer/src/macros.rs b/observability/crates/event_indexer/src/macros.rs new file mode 100644 index 00000000..b6bd5c7d --- /dev/null +++ b/observability/crates/event_indexer/src/macros.rs @@ -0,0 +1,22 @@ +#[macro_export] +macro_rules! insert_if_needed { + ($connection:ident, $table_name:ident, $model:ident, $record_address:expr, $record_fields:expr) => {{ + let records = $table_name::dsl::$table_name + .filter($table_name::address.eq($record_address)) + .select($model::as_select()) + .limit(1) + .load($connection)?; + + let record_id = if records.len() == 0 { + diesel::insert_into($table_name::table) + .values(vec![$record_fields]) + .on_conflict_do_nothing() + .returning($table_name::id) + .get_result($connection)? + } else { + records.first().unwrap().id + }; + + record_id + }}; +} diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 61760eef..27bc7be1 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -32,6 +32,7 @@ use crate::{ db::{models::*, schema::*}, entity_store::EntityStore, error::IndexingError, + insert_if_needed, }; const SPL_TRANSFER_DISCRIMINATOR: u8 = 3; @@ -94,44 +95,28 @@ impl MarginfiEvent for CreateAccountEvent { ) -> Result<(), IndexingError> { db_connection .transaction(|connection: &mut PgConnection| { - let users = users::dsl::users - .filter(users::address.eq(self.authority.to_string())) - .select(Users::as_select()) - .limit(1) - .load(connection)?; - - let authority_id = if users.len() == 0 { - diesel::insert_into(users::table) - .values(vec![Users { - address: self.authority.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(users::id) - .get_result(connection)? - } else { - users.first().unwrap().id - }; + let authority_id = insert_if_needed!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ); - let accounts = accounts::dsl::accounts - .filter(accounts::address.eq(self.account.to_string())) - .select(Accounts::as_select()) - .limit(1) - .load(connection)?; - - let account_id = if accounts.len() == 0 { - diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(accounts::id) - .get_result(connection)? - } else { - accounts.first().unwrap().id - }; + let account_id = insert_if_needed!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ); let create_account_event = CreateAccountEvents { timestamp, @@ -172,63 +157,39 @@ impl MarginfiEvent for AccountAuthorityTransferEvent { ) -> Result<(), IndexingError> { db_connection .transaction(|connection: &mut PgConnection| { - let users = users::dsl::users - .filter(users::address.eq(self.old_authority.to_string())) - .select(Users::as_select()) - .limit(1) - .load(connection)?; - - let old_authority_id = if users.len() == 0 { - diesel::insert_into(users::table) - .values(vec![Users { - address: self.old_authority.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(users::id) - .get_result(connection)? - } else { - users.first().unwrap().id - }; + let old_authority_id = insert_if_needed!( + connection, + users, + Users, + self.old_authority.to_string(), + Users { + address: self.old_authority.to_string(), + ..Default::default() + } + ); - let users = users::dsl::users - .filter(users::address.eq(self.new_authority.to_string())) - .select(Users::as_select()) - .limit(1) - .load(connection)?; - - let new_authority_id = if users.len() == 0 { - diesel::insert_into(users::table) - .values(vec![Users { - address: self.new_authority.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(users::id) - .get_result(connection)? - } else { - users.first().unwrap().id - }; + let new_authority_id = insert_if_needed!( + connection, + users, + Users, + self.new_authority.to_string(), + Users { + address: self.new_authority.to_string(), + ..Default::default() + } + ); - let accounts = accounts::dsl::accounts - .filter(accounts::address.eq(self.account.to_string())) - .select(Accounts::as_select()) - .limit(1) - .load(connection)?; - - let account_id = if accounts.len() == 0 { - diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: self.account.to_string(), - user_id: new_authority_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(accounts::id) - .get_result(connection)? - } else { - accounts.first().unwrap().id - }; + let account_id = insert_if_needed!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: new_authority_id, + ..Default::default() + } + ); let account_authority_transfer_event = TransferAccountAuthorityEvents { timestamp, @@ -269,91 +230,57 @@ impl MarginfiEvent for DepositEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { - let bank_data = entity_store - .get_or_fetch_bank(&self.bank.to_string()) - .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let users = users::dsl::users - .filter(users::address.eq(self.authority.to_string())) - .select(Users::as_select()) - .limit(1) - .load(connection)?; - - let authority_id = if users.len() == 0 { - diesel::insert_into(users::table) - .values(vec![Users { - address: self.authority.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(users::id) - .get_result(connection)? - } else { - users.first().unwrap().id - }; + let authority_id = insert_if_needed!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ); - let accounts = accounts::dsl::accounts - .filter(accounts::address.eq(self.account.to_string())) - .select(Accounts::as_select()) - .limit(1) - .load(connection)?; - - let account_id = if accounts.len() == 0 { - diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(accounts::id) - .get_result(connection)? - } else { - accounts.first().unwrap().id - }; + let account_id = insert_if_needed!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ); - let mints = mints::dsl::mints - .filter(mints::address.eq(bank_data.mint.address.clone())) - .select(Mints::as_select()) - .limit(1) - .load(connection)?; - - let mint_id = if mints.len() == 0 { - diesel::insert_into(mints::table) - .values(vec![Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(mints::id) - .get_result(connection)? - } else { - mints.first().unwrap().id - }; + let bank_mint_id = insert_if_needed!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ); - let banks = banks::dsl::banks - .filter(banks::address.eq(self.bank.to_string())) - .select(Banks::as_select()) - .limit(1) - .load(connection)?; - - let bank_id = if banks.len() == 0 { - diesel::insert_into(banks::table) - .values(vec![Banks { - address: self.bank.to_string(), - mint_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(banks::id) - .get_result(connection)? - } else { - banks.first().unwrap().id - }; + let bank_id = insert_if_needed!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ); let deposit_event = DepositEvents { timestamp, @@ -395,91 +322,57 @@ impl MarginfiEvent for BorrowEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { - let bank_data = entity_store - .get_or_fetch_bank(&self.bank.to_string()) - .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let users = users::dsl::users - .filter(users::address.eq(self.authority.to_string())) - .select(Users::as_select()) - .limit(1) - .load(connection)?; - - let authority_id = if users.len() == 0 { - diesel::insert_into(users::table) - .values(vec![Users { - address: self.authority.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(users::id) - .get_result(connection)? - } else { - users.first().unwrap().id - }; + let authority_id = insert_if_needed!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ); - let accounts = accounts::dsl::accounts - .filter(accounts::address.eq(self.account.to_string())) - .select(Accounts::as_select()) - .limit(1) - .load(connection)?; - - let account_id = if accounts.len() == 0 { - diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(accounts::id) - .get_result(connection)? - } else { - accounts.first().unwrap().id - }; + let account_id = insert_if_needed!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ); - let mints = mints::dsl::mints - .filter(mints::address.eq(bank_data.mint.address.clone())) - .select(Mints::as_select()) - .limit(1) - .load(connection)?; - - let mint_id = if mints.len() == 0 { - diesel::insert_into(mints::table) - .values(vec![Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(mints::id) - .get_result(connection)? - } else { - mints.first().unwrap().id - }; + let bank_mint_id = insert_if_needed!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ); - let banks = banks::dsl::banks - .filter(banks::address.eq(self.bank.to_string())) - .select(Banks::as_select()) - .limit(1) - .load(connection)?; - - let bank_id = if banks.len() == 0 { - diesel::insert_into(banks::table) - .values(vec![Banks { - address: self.bank.to_string(), - mint_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(banks::id) - .get_result(connection)? - } else { - banks.first().unwrap().id - }; + let bank_id = insert_if_needed!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ); let borrow_event = BorrowEvents { timestamp, @@ -522,91 +415,57 @@ impl MarginfiEvent for RepayEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { - let bank_data = entity_store - .get_or_fetch_bank(&self.bank.to_string()) - .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let users = users::dsl::users - .filter(users::address.eq(self.authority.to_string())) - .select(Users::as_select()) - .limit(1) - .load(connection)?; - - let authority_id = if users.len() == 0 { - diesel::insert_into(users::table) - .values(vec![Users { - address: self.authority.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(users::id) - .get_result(connection)? - } else { - users.first().unwrap().id - }; + let authority_id = insert_if_needed!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ); - let accounts = accounts::dsl::accounts - .filter(accounts::address.eq(self.account.to_string())) - .select(Accounts::as_select()) - .limit(1) - .load(connection)?; - - let account_id = if accounts.len() == 0 { - diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(accounts::id) - .get_result(connection)? - } else { - accounts.first().unwrap().id - }; + let account_id = insert_if_needed!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ); - let mints = mints::dsl::mints - .filter(mints::address.eq(bank_data.mint.address.clone())) - .select(Mints::as_select()) - .limit(1) - .load(connection)?; - - let mint_id = if mints.len() == 0 { - diesel::insert_into(mints::table) - .values(vec![Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(mints::id) - .get_result(connection)? - } else { - mints.first().unwrap().id - }; + let bank_mint_id = insert_if_needed!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ); - let banks = banks::dsl::banks - .filter(banks::address.eq(self.bank.to_string())) - .select(Banks::as_select()) - .limit(1) - .load(connection)?; - - let bank_id = if banks.len() == 0 { - diesel::insert_into(banks::table) - .values(vec![Banks { - address: self.bank.to_string(), - mint_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(banks::id) - .get_result(connection)? - } else { - banks.first().unwrap().id - }; + let bank_id = insert_if_needed!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ); let repay_event = RepayEvents { timestamp, @@ -650,91 +509,57 @@ impl MarginfiEvent for WithdrawEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { - let bank_data = entity_store - .get_or_fetch_bank(&self.bank.to_string()) - .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let users = users::dsl::users - .filter(users::address.eq(self.authority.to_string())) - .select(Users::as_select()) - .limit(1) - .load(connection)?; - - let authority_id = if users.len() == 0 { - diesel::insert_into(users::table) - .values(vec![Users { - address: self.authority.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(users::id) - .get_result(connection)? - } else { - users.first().unwrap().id - }; + let authority_id = insert_if_needed!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ); - let accounts = accounts::dsl::accounts - .filter(accounts::address.eq(self.account.to_string())) - .select(Accounts::as_select()) - .limit(1) - .load(connection)?; - - let account_id = if accounts.len() == 0 { - diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(accounts::id) - .get_result(connection)? - } else { - accounts.first().unwrap().id - }; + let account_id = insert_if_needed!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ); - let mints = mints::dsl::mints - .filter(mints::address.eq(bank_data.mint.address.clone())) - .select(Mints::as_select()) - .limit(1) - .load(connection)?; - - let mint_id = if mints.len() == 0 { - diesel::insert_into(mints::table) - .values(vec![Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(mints::id) - .get_result(connection)? - } else { - mints.first().unwrap().id - }; + let bank_mint_id = insert_if_needed!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ); - let banks = banks::dsl::banks - .filter(banks::address.eq(self.bank.to_string())) - .select(Banks::as_select()) - .limit(1) - .load(connection)?; - - let bank_id = if banks.len() == 0 { - diesel::insert_into(banks::table) - .values(vec![Banks { - address: self.bank.to_string(), - mint_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(banks::id) - .get_result(connection)? - } else { - banks.first().unwrap().id - }; + let bank_id = insert_if_needed!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ); let withdraw_event = WithdrawEvents { timestamp, @@ -778,115 +603,73 @@ impl MarginfiEvent for WithdrawEmissionsEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { - let bank_data = entity_store - .get_or_fetch_bank(&self.bank.to_string()) - .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; let emission_mint_data = entity_store.get_or_fetch_mint(&self.emissions_mint.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let users = users::dsl::users - .filter(users::address.eq(self.authority.to_string())) - .select(Users::as_select()) - .limit(1) - .load(connection)?; - - let authority_id = if users.len() == 0 { - diesel::insert_into(users::table) - .values(vec![Users { - address: self.authority.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(users::id) - .get_result(connection)? - } else { - users.first().unwrap().id - }; + let authority_id = insert_if_needed!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ); - let accounts = accounts::dsl::accounts - .filter(accounts::address.eq(self.account.to_string())) - .select(Accounts::as_select()) - .limit(1) - .load(connection)?; - - let account_id = if accounts.len() == 0 { - diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(accounts::id) - .get_result(connection)? - } else { - accounts.first().unwrap().id - }; + let account_id = insert_if_needed!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ); - let mints = mints::dsl::mints - .filter(mints::address.eq(bank_data.mint.address.clone())) - .select(Mints::as_select()) - .limit(1) - .load(connection)?; - - let mint_id = if mints.len() == 0 { - diesel::insert_into(mints::table) - .values(vec![Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(mints::id) - .get_result(connection)? - } else { - mints.first().unwrap().id - }; + let bank_mint_id = insert_if_needed!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ); - let banks = banks::dsl::banks - .filter(banks::address.eq(self.bank.to_string())) - .select(Banks::as_select()) - .limit(1) - .load(connection)?; - - let bank_id = if banks.len() == 0 { - diesel::insert_into(banks::table) - .values(vec![Banks { - address: self.bank.to_string(), - mint_id, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(banks::id) - .get_result(connection)? - } else { - banks.first().unwrap().id - }; + let emission_mint_id = insert_if_needed!( + connection, + mints, + Mints, + self.emissions_mint.to_string(), + Mints { + address: self.emissions_mint.to_string(), + symbol: emission_mint_data.symbol.clone(), + decimals: emission_mint_data.decimals, + ..Default::default() + } + ); - let emissions_mints = mints::dsl::mints - .filter(mints::address.eq(self.emissions_mint.to_string())) - .select(Mints::as_select()) - .limit(1) - .load(connection)?; - - let emission_mint_id = if emissions_mints.len() == 0 { - diesel::insert_into(mints::table) - .values(vec![Mints { - address: self.emissions_mint.to_string(), - symbol: emission_mint_data.symbol.clone(), - decimals: emission_mint_data.decimals, - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(mints::id) - .get_result(connection)? - } else { - emissions_mints.first().unwrap().id - }; + let bank_id = insert_if_needed!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ); let withdraw_emissions_event = WithdrawEmissionsEvents { timestamp, @@ -931,110 +714,111 @@ impl MarginfiEvent for LiquidateEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { - let asset_bank_data = entity_store - .get_or_fetch_bank(&self.asset_bank.to_string()) - .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + let asset_bank_data = entity_store.get_or_fetch_bank(&self.asset_bank.to_string())?; - let liability_bank_data = entity_store - .get_or_fetch_bank(&self.liability_bank.to_string()) - .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + let liability_bank_data = + entity_store.get_or_fetch_bank(&self.liability_bank.to_string())?; + + let liquidatee_account_data = + entity_store.get_or_fetch_account(&self.liquidatee_account.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let users = users::dsl::users - .filter(users::address.eq(self.liquidator_authority.to_string())) - .select(Users::as_select()) - .limit(1) - .load(connection)?; - - let liquidator_user_id = if users.len() == 0 { - diesel::insert_into(users::table) - .values(vec![Users { - address: self.liquidator_authority.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(users::id) - .get_result(connection)? - } else { - users.first().unwrap().id - }; + let liquidator_user_id = insert_if_needed!( + connection, + users, + Users, + self.liquidator_authority.to_string(), + Users { + address: self.liquidator_authority.to_string(), + ..Default::default() + } + ); - let accounts = accounts::dsl::accounts - .filter(accounts::address.eq(self.liquidator_account.to_string())) - .select(Accounts::as_select()) - .limit(1) - .load(connection)?; - - let liquidator_account_id = if accounts.len() == 0 { - diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: self.liquidator_account.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(accounts::id) - .get_result(connection)? - } else { - accounts.first().unwrap().id - }; + let liquidatee_user_id = insert_if_needed!( + connection, + users, + Users, + liquidatee_account_data.authority.clone(), + Users { + address: liquidatee_account_data.authority.clone(), + ..Default::default() + } + ); - let accounts = accounts::dsl::accounts - .filter(accounts::address.eq(self.liquidatee_account.to_string())) - .select(Accounts::as_select()) - .limit(1) - .load(connection)?; - - let liquidatee_account_id = if accounts.len() == 0 { - diesel::insert_into(accounts::table) - .values(vec![Accounts { - address: self.liquidatee_account.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(accounts::id) - .get_result(connection)? - } else { - accounts.first().unwrap().id - }; + let liquidator_account_id = insert_if_needed!( + connection, + accounts, + Accounts, + self.liquidator_account.to_string(), + Accounts { + address: self.liquidator_account.to_string(), + user_id: liquidator_user_id, + ..Default::default() + } + ); - let banks = banks::dsl::banks - .filter(banks::address.eq(self.asset_bank.to_string())) - .select(Banks::as_select()) - .limit(1) - .load(connection)?; - - let asset_bank_id = if banks.len() == 0 { - diesel::insert_into(banks::table) - .values(vec![Banks { - address: self.asset_bank.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(banks::id) - .get_result(connection)? - } else { - banks.first().unwrap().id - }; + let liquidatee_account_id = insert_if_needed!( + connection, + accounts, + Accounts, + self.liquidatee_account.to_string(), + Accounts { + address: self.liquidatee_account.to_string(), + user_id: liquidatee_user_id, + ..Default::default() + } + ); - let banks = banks::dsl::banks - .filter(banks::address.eq(self.liability_bank.to_string())) - .select(Banks::as_select()) - .limit(1) - .load(connection)?; - - let liability_bank_id = if banks.len() == 0 { - diesel::insert_into(banks::table) - .values(vec![Banks { - address: self.liability_bank.to_string(), - ..Default::default() - }]) - .on_conflict_do_nothing() - .returning(banks::id) - .get_result(connection)? - } else { - banks.first().unwrap().id - }; + let asset_mint_id = insert_if_needed!( + connection, + mints, + Mints, + asset_bank_data.mint.address.clone(), + Mints { + address: asset_bank_data.mint.address.clone(), + symbol: asset_bank_data.mint.symbol.clone(), + decimals: asset_bank_data.mint.decimals, + ..Default::default() + } + ); + + let liability_mint_id = insert_if_needed!( + connection, + mints, + Mints, + liability_bank_data.mint.address.clone(), + Mints { + address: liability_bank_data.mint.address.clone(), + symbol: liability_bank_data.mint.symbol.clone(), + decimals: liability_bank_data.mint.decimals, + ..Default::default() + } + ); + + let asset_bank_id = insert_if_needed!( + connection, + banks, + Banks, + self.asset_bank.to_string(), + Banks { + address: self.asset_bank.to_string(), + mint_id: asset_mint_id, + ..Default::default() + } + ); + + let liability_bank_id = insert_if_needed!( + connection, + banks, + Banks, + self.liability_bank.to_string(), + Banks { + address: self.liability_bank.to_string(), + mint_id: liability_mint_id, + ..Default::default() + } + ); let liquidate_event = LiquidateEvents { timestamp, From 844119e808b84e114fbd6113c830f05d52ec754a Mon Sep 17 00:00:00 2001 From: losman0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 28 Mar 2024 23:25:47 +0800 Subject: [PATCH 06/20] feat: user actions live ingest --- .../2024-03-14-151141_initial_tables/up.sql | 2 +- .../crates/event_indexer/src/db/models.rs | 1 + .../crates/event_indexer/src/entity_store.rs | 194 +++- .../crates/event_indexer/src/indexer.rs | 8 +- .../crates/event_indexer/src/macros.rs | 26 +- .../crates/event_indexer/src/parser.rs | 889 ++++++++++-------- 6 files changed, 693 insertions(+), 427 deletions(-) diff --git a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql index 64943bdf..7e04c35d 100644 --- a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql +++ b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql @@ -2,7 +2,7 @@ CREATE TABLE "mints"( "id" SERIAL PRIMARY KEY, - "address" VARCHAR NOT NULL, + "address" VARCHAR NOT NULL UNIQUE, "symbol" VARCHAR NOT NULL, "decimals" SMALLINT NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), diff --git a/observability/crates/event_indexer/src/db/models.rs b/observability/crates/event_indexer/src/db/models.rs index 79716039..92fc77af 100644 --- a/observability/crates/event_indexer/src/db/models.rs +++ b/observability/crates/event_indexer/src/db/models.rs @@ -13,6 +13,7 @@ pub struct Mints { } #[derive(Default, Debug, Queryable, Selectable, Insertable)] +#[diesel(belongs_to(Mints, foreign_key = mint_id))] #[diesel(table_name = banks)] pub struct Banks { #[diesel(skip_insertion)] diff --git a/observability/crates/event_indexer/src/entity_store.rs b/observability/crates/event_indexer/src/entity_store.rs index ebecbbc6..69b01027 100644 --- a/observability/crates/event_indexer/src/entity_store.rs +++ b/observability/crates/event_indexer/src/entity_store.rs @@ -24,7 +24,7 @@ pub struct EntityStore { mint_cache: HashMap, bank_cache: HashMap, account_cache: HashMap, - authority_cache: HashMap, + user_cache: HashMap, } impl EntityStore { @@ -44,7 +44,7 @@ impl EntityStore { mint_cache: HashMap::new(), bank_cache: HashMap::new(), account_cache: HashMap::new(), - authority_cache: HashMap::new(), + user_cache: HashMap::new(), } } @@ -54,7 +54,12 @@ impl EntityStore { if let Some(mint) = maybe_mint { Ok(mint.clone()) } else { - let mint = MintData::fetch(&self.rpc_client, &mut self.db_connection, address)?; + let mint = MintData::fetch(self, address)?; + + if mint.id.is_some() { + self.mint_cache.insert(address.to_string(), mint.clone()); + } + Ok(mint) } } @@ -65,21 +70,48 @@ impl EntityStore { if let Some(bank) = maybe_bank { Ok(bank.clone()) } else { - let bank = BankData::fetch(&self.rpc_client, &mut self.db_connection, address)?; + let bank = BankData::fetch(self, address)?; + + if bank.id.is_some() { + self.bank_cache.insert(address.to_string(), bank.clone()); + } + Ok(bank) } } pub fn get_or_fetch_account(&mut self, address: &str) -> Result { - let maybe_account = self.account_cache.get(&address.to_string()); + let maybe_account = { self.account_cache.get(&address.to_string()).cloned() }; if let Some(account) = maybe_account { Ok(account.clone()) } else { - let account = AccountData::fetch(&self.rpc_client, &mut self.db_connection, address)?; + let account = AccountData::fetch(self, address)?; + + if account.id.is_some() { + self.account_cache + .insert(address.to_string(), account.clone()); + } + Ok(account) } } + + pub fn get_or_fetch_user(&mut self, address: &str) -> Result { + let maybe_user = self.user_cache.get(&address.to_string()); + + if let Some(user) = maybe_user { + Ok(user.clone()) + } else { + let user = UserData::fetch(&mut self.db_connection, address)?; + + if user.id.is_some() { + self.user_cache.insert(address.to_string(), user.clone()); + } + + Ok(user) + } + } } #[derive(Debug, Clone)] @@ -91,31 +123,27 @@ pub struct MintData { } impl MintData { - pub fn fetch( - rpc_client: &RpcClient, - db_connection: &mut PgConnection, - address: &str, - ) -> Result { - let db_record = Self::fetch_from_db(db_connection, address)?; + pub fn fetch(entity_store: &mut EntityStore, address: &str) -> Result { + let db_record = Self::fetch_from_db(entity_store, address)?; if let Some(db_record) = db_record { return Ok(db_record); } - let mint = Self::fetch_from_rpc(rpc_client, address)?; + let mint = Self::fetch_from_rpc(&entity_store.rpc_client, address)?; Ok(mint) } fn fetch_from_db( - db_connection: &mut PgConnection, + entity_store: &mut EntityStore, address: &str, ) -> Result, IndexingError> { let db_records = mints::dsl::mints .filter(mints::address.eq(address.to_string())) .select(Mints::as_select()) .limit(1) - .load(db_connection) + .load(&mut entity_store.db_connection) .map_err(|e| { IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( "mint".to_string(), @@ -171,31 +199,27 @@ pub struct BankData { } impl BankData { - pub fn fetch( - rpc_client: &RpcClient, - db_connection: &mut PgConnection, - address: &str, - ) -> Result { - let db_record = Self::fetch_from_db(db_connection, address)?; + pub fn fetch(entity_store: &mut EntityStore, address: &str) -> Result { + let db_record = Self::fetch_from_db(entity_store, address)?; if let Some(db_record) = db_record { return Ok(db_record); } - let mint = Self::fetch_from_rpc(rpc_client, db_connection, address)?; + let mint = Self::fetch_from_rpc(entity_store, address)?; Ok(mint) } fn fetch_from_db( - db_connection: &mut PgConnection, + entity_store: &mut EntityStore, address: &str, ) -> Result, IndexingError> { let db_records = banks::dsl::banks .filter(banks::address.eq(address.to_string())) .select(Banks::as_select()) .limit(1) - .load(db_connection) + .load(&mut entity_store.db_connection) .map_err(|e| { IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( "bank".to_string(), @@ -209,23 +233,41 @@ impl BankData { let db_record = db_records.get(0).unwrap(); - let mint_data = MintData::fetch_from_db(db_connection, &db_record.mint_id.to_string()) - .unwrap() - .unwrap(); + let mint = mints::dsl::mints + .find(&db_record.mint_id) + .select(Mints::as_select()) + .first(&mut entity_store.db_connection) + .map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( + "mint".to_string(), + e.to_string(), + )) + })?; + + let mint_data = if let Some(mint_data) = entity_store.mint_cache.get(&mint.address) { + mint_data.clone() + } else { + MintData::fetch(entity_store, &mint.address)? + }; Ok(Some(Self { id: Some(db_record.id), address: db_record.address.clone(), - mint: mint_data, + mint: MintData { + id: mint_data.id, + address: mint_data.address, + symbol: mint_data.symbol, + decimals: mint_data.decimals, + }, })) } fn fetch_from_rpc( - rpc_client: &RpcClient, - db_connection: &mut PgConnection, + entity_store: &mut EntityStore, address: &str, ) -> Result { - let mint_data = rpc_client + let data = entity_store + .rpc_client .get_account_data(&Pubkey::from_str(address).unwrap()) .map_err(|e| { IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( @@ -234,14 +276,14 @@ impl BankData { )) })?; - let bank = Bank::try_deserialize(&mut mint_data.as_slice()).map_err(|e| { + let bank = Bank::try_deserialize(&mut data.as_slice()).map_err(|e| { IndexingError::FailedToFetchEntity(FetchEntityError::UnpackError( "bank".to_string(), e.to_string(), )) })?; - let mint_data = MintData::fetch(rpc_client, db_connection, &bank.mint.to_string())?; + let mint_data = MintData::fetch(entity_store, &bank.mint.to_string())?; Ok(Self { id: None, @@ -255,34 +297,30 @@ impl BankData { pub struct AccountData { pub id: Option, pub address: String, - pub authority: String, + pub authority: UserData, } impl AccountData { - pub fn fetch( - rpc_client: &RpcClient, - db_connection: &mut PgConnection, - address: &str, - ) -> Result { - let db_record = Self::fetch_from_db(db_connection, address)?; + pub fn fetch(entity_store: &mut EntityStore, address: &str) -> Result { + let db_record = Self::fetch_from_db(entity_store, address)?; if let Some(db_record) = db_record { return Ok(db_record); } - let mint = Self::fetch_from_rpc(rpc_client, address)?; + let mint = Self::fetch_from_rpc(entity_store, address)?; Ok(mint) } fn fetch_from_db( - db_connection: &mut PgConnection, + entity_store: &mut EntityStore, address: &str, ) -> Result, IndexingError> { let maybe_db_record = accounts::dsl::accounts .filter(accounts::address.eq(address.to_string())) .select(Accounts::as_select()) - .get_result(db_connection) + .get_result(&mut entity_store.db_connection) .optional() .map_err(|e| { IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( @@ -300,7 +338,7 @@ impl AccountData { let authority = users::dsl::users .find(&db_record.user_id) .select(Users::as_select()) - .first(db_connection) + .first(&mut entity_store.db_connection) .map_err(|e| { IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( "account".to_string(), @@ -308,15 +346,25 @@ impl AccountData { )) })?; + let user_data = if let Some(user_data) = entity_store.user_cache.get(&authority.address) { + user_data.clone() + } else { + UserData::fetch(&mut entity_store.db_connection, address)? + }; + Ok(Some(Self { id: Some(db_record.id), address: db_record.address.clone(), - authority: authority.address.clone(), + authority: user_data, })) } - fn fetch_from_rpc(rpc_client: &RpcClient, address: &str) -> Result { - let mint_data = rpc_client + fn fetch_from_rpc( + entity_store: &mut EntityStore, + address: &str, + ) -> Result { + let data = entity_store + .rpc_client .get_account_data(&Pubkey::from_str(address).unwrap()) .map_err(|e| { IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( @@ -325,17 +373,24 @@ impl AccountData { )) })?; - let account = MarginfiAccount::try_deserialize(&mut mint_data.as_slice()).map_err(|e| { + let account = MarginfiAccount::try_deserialize(&mut data.as_slice()).map_err(|e| { IndexingError::FailedToFetchEntity(FetchEntityError::UnpackError( "account".to_string(), e.to_string(), )) })?; + let user_data = + if let Some(user_data) = entity_store.user_cache.get(&account.authority.to_string()) { + user_data.clone() + } else { + UserData::fetch(&mut entity_store.db_connection, address)? + }; + Ok(Self { id: None, address: address.to_string(), - authority: account.authority.to_string(), + authority: user_data, }) } } @@ -347,7 +402,44 @@ pub struct UserData { } impl UserData { - fn get_cache_or_fetch(&mut self, address: &str) -> UserData { - todo!() + pub fn fetch(db_connection: &mut PgConnection, address: &str) -> Result { + let db_record = Self::fetch_from_db(db_connection, address)?; + + if let Some(db_record) = db_record { + return Ok(db_record); + } + + Ok(Self { + id: None, + address: address.to_string(), + }) + } + + fn fetch_from_db( + db_connection: &mut PgConnection, + address: &str, + ) -> Result, IndexingError> { + let maybe_db_record = users::dsl::users + .filter(users::address.eq(address.to_string())) + .select(Users::as_select()) + .get_result(db_connection) + .optional() + .map_err(|e| { + IndexingError::FailedToFetchEntity(FetchEntityError::FetchError( + "user".to_string(), + e.to_string(), + )) + })?; + + if maybe_db_record.is_none() { + return Ok(None); + } + + let db_record = maybe_db_record.unwrap(); + + Ok(Some(Self { + id: Some(db_record.id), + address: db_record.address.clone(), + })) } } diff --git a/observability/crates/event_indexer/src/indexer.rs b/observability/crates/event_indexer/src/indexer.rs index dfa30026..6fa718df 100644 --- a/observability/crates/event_indexer/src/indexer.rs +++ b/observability/crates/event_indexer/src/indexer.rs @@ -289,7 +289,13 @@ async fn store_events( { let timestamp = DateTime::from_timestamp(timestamp, 0).unwrap().naive_utc(); let tx_sig = tx_sig.to_string(); - let call_stack = serde_json::to_string(&call_stack).unwrap(); + let call_stack = serde_json::to_string( + &call_stack + .into_iter() + .map(|cs| cs.to_string()) + .collect::>(), + ) + .unwrap_or_else(|_| "null".to_string()); println!("Storing event: {:?}", event); event .db_insert( diff --git a/observability/crates/event_indexer/src/macros.rs b/observability/crates/event_indexer/src/macros.rs index b6bd5c7d..ec13bd80 100644 --- a/observability/crates/event_indexer/src/macros.rs +++ b/observability/crates/event_indexer/src/macros.rs @@ -1,5 +1,5 @@ #[macro_export] -macro_rules! insert_if_needed { +macro_rules! get_and_insert_if_needed { ($connection:ident, $table_name:ident, $model:ident, $record_address:expr, $record_fields:expr) => {{ let records = $table_name::dsl::$table_name .filter($table_name::address.eq($record_address)) @@ -8,11 +8,13 @@ macro_rules! insert_if_needed { .load($connection)?; let record_id = if records.len() == 0 { - diesel::insert_into($table_name::table) - .values(vec![$record_fields]) - .on_conflict_do_nothing() - .returning($table_name::id) - .get_result($connection)? + insert!( + $connection, + $table_name, + $model, + $record_address, + $record_fields + ) } else { records.first().unwrap().id }; @@ -20,3 +22,15 @@ macro_rules! insert_if_needed { record_id }}; } + +#[macro_export] +macro_rules! insert { + ($connection:ident, $table_name:ident, $model:ident, $record_address:expr, $record_fields:expr) => {{ + let record_id = diesel::insert_into($table_name::table) + .values(vec![$record_fields]) + .returning($table_name::id) + .get_result($connection)?; + + record_id + }}; +} diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 27bc7be1..91224f45 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -32,7 +32,7 @@ use crate::{ db::{models::*, schema::*}, entity_store::EntityStore, error::IndexingError, - insert_if_needed, + get_and_insert_if_needed, insert, }; const SPL_TRANSFER_DISCRIMINATOR: u8 = 3; @@ -91,22 +91,29 @@ impl MarginfiEvent for CreateAccountEvent { in_flashloan: bool, call_stack: String, db_connection: &mut PgConnection, - _entity_store: &mut EntityStore, + entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let authority_data = entity_store.get_or_fetch_user(&self.authority.to_string())?; + db_connection .transaction(|connection: &mut PgConnection| { - let authority_id = insert_if_needed!( - connection, - users, - Users, - self.authority.to_string(), - Users { - address: self.authority.to_string(), - ..Default::default() - } - ); + let authority_id = if let Some(id) = authority_data.id { + id + } else { + insert!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ) + }; - let account_id = insert_if_needed!( + // Not RPC fetching the account data here because it could lead to race condition with the RPC when live ingesting, + let account_id = get_and_insert_if_needed!( connection, accounts, Accounts, @@ -153,43 +160,59 @@ impl MarginfiEvent for AccountAuthorityTransferEvent { in_flashloan: bool, call_stack: String, db_connection: &mut PgConnection, - _entity_store: &mut EntityStore, + entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let old_authority_data = entity_store.get_or_fetch_user(&self.old_authority.to_string())?; + let new_authority_data = entity_store.get_or_fetch_user(&self.new_authority.to_string())?; + let account_data = entity_store.get_or_fetch_account(&self.account.to_string())?; + db_connection .transaction(|connection: &mut PgConnection| { - let old_authority_id = insert_if_needed!( - connection, - users, - Users, - self.old_authority.to_string(), - Users { - address: self.old_authority.to_string(), - ..Default::default() - } - ); + let old_authority_id = if let Some(id) = old_authority_data.id { + id + } else { + insert!( + connection, + users, + Users, + self.old_authority.to_string(), + Users { + address: self.old_authority.to_string(), + ..Default::default() + } + ) + }; - let new_authority_id = insert_if_needed!( - connection, - users, - Users, - self.new_authority.to_string(), - Users { - address: self.new_authority.to_string(), - ..Default::default() - } - ); + let new_authority_id = if let Some(id) = new_authority_data.id { + id + } else { + insert!( + connection, + users, + Users, + self.new_authority.to_string(), + Users { + address: self.new_authority.to_string(), + ..Default::default() + } + ) + }; - let account_id = insert_if_needed!( - connection, - accounts, - Accounts, - self.account.to_string(), - Accounts { - address: self.account.to_string(), - user_id: new_authority_id, - ..Default::default() - } - ); + let account_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: new_authority_id, + ..Default::default() + } + ) + }; let account_authority_transfer_event = TransferAccountAuthorityEvents { timestamp, @@ -230,57 +253,75 @@ impl MarginfiEvent for DepositEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let authority_data = entity_store.get_or_fetch_user(&self.authority.to_string())?; + let account_data = entity_store.get_or_fetch_account(&self.account.to_string())?; let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let authority_id = insert_if_needed!( - connection, - users, - Users, - self.authority.to_string(), - Users { - address: self.authority.to_string(), - ..Default::default() - } - ); + let authority_id = if let Some(id) = authority_data.id { + id + } else { + insert!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ) + }; - let account_id = insert_if_needed!( - connection, - accounts, - Accounts, - self.account.to_string(), - Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - } - ); + let account_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ) + }; - let bank_mint_id = insert_if_needed!( - connection, - mints, - Mints, - bank_data.mint.address.clone(), - Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - } - ); + let bank_mint_id = if let Some(id) = bank_data.mint.id { + id + } else { + insert!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ) + }; - let bank_id = insert_if_needed!( - connection, - banks, - Banks, - self.bank.to_string(), - Banks { - address: self.bank.to_string(), - mint_id: bank_mint_id, - ..Default::default() - } - ); + let bank_id = if let Some(id) = bank_data.id { + id + } else { + insert!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ) + }; let deposit_event = DepositEvents { timestamp, @@ -322,57 +363,75 @@ impl MarginfiEvent for BorrowEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let authority_data = entity_store.get_or_fetch_user(&self.authority.to_string())?; + let account_data = entity_store.get_or_fetch_account(&self.account.to_string())?; let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let authority_id = insert_if_needed!( - connection, - users, - Users, - self.authority.to_string(), - Users { - address: self.authority.to_string(), - ..Default::default() - } - ); + let authority_id = if let Some(id) = authority_data.id { + id + } else { + insert!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ) + }; - let account_id = insert_if_needed!( - connection, - accounts, - Accounts, - self.account.to_string(), - Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - } - ); + let account_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ) + }; - let bank_mint_id = insert_if_needed!( - connection, - mints, - Mints, - bank_data.mint.address.clone(), - Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - } - ); + let bank_mint_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ) + }; - let bank_id = insert_if_needed!( - connection, - banks, - Banks, - self.bank.to_string(), - Banks { - address: self.bank.to_string(), - mint_id: bank_mint_id, - ..Default::default() - } - ); + let bank_id = if let Some(id) = bank_data.id { + id + } else { + insert!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ) + }; let borrow_event = BorrowEvents { timestamp, @@ -415,57 +474,75 @@ impl MarginfiEvent for RepayEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let authority_data = entity_store.get_or_fetch_user(&self.authority.to_string())?; + let account_data = entity_store.get_or_fetch_account(&self.account.to_string())?; let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let authority_id = insert_if_needed!( - connection, - users, - Users, - self.authority.to_string(), - Users { - address: self.authority.to_string(), - ..Default::default() - } - ); + let authority_id = if let Some(id) = authority_data.id { + id + } else { + insert!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ) + }; - let account_id = insert_if_needed!( - connection, - accounts, - Accounts, - self.account.to_string(), - Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - } - ); + let account_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ) + }; - let bank_mint_id = insert_if_needed!( - connection, - mints, - Mints, - bank_data.mint.address.clone(), - Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - } - ); + let bank_mint_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ) + }; - let bank_id = insert_if_needed!( - connection, - banks, - Banks, - self.bank.to_string(), - Banks { - address: self.bank.to_string(), - mint_id: bank_mint_id, - ..Default::default() - } - ); + let bank_id = if let Some(id) = bank_data.id { + id + } else { + insert!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ) + }; let repay_event = RepayEvents { timestamp, @@ -509,57 +586,75 @@ impl MarginfiEvent for WithdrawEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let authority_data = entity_store.get_or_fetch_user(&self.authority.to_string())?; + let account_data = entity_store.get_or_fetch_account(&self.account.to_string())?; let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let authority_id = insert_if_needed!( - connection, - users, - Users, - self.authority.to_string(), - Users { - address: self.authority.to_string(), - ..Default::default() - } - ); + let authority_id = if let Some(id) = authority_data.id { + id + } else { + insert!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ) + }; - let account_id = insert_if_needed!( - connection, - accounts, - Accounts, - self.account.to_string(), - Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - } - ); + let account_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ) + }; - let bank_mint_id = insert_if_needed!( - connection, - mints, - Mints, - bank_data.mint.address.clone(), - Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - } - ); + let bank_mint_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ) + }; - let bank_id = insert_if_needed!( - connection, - banks, - Banks, - self.bank.to_string(), - Banks { - address: self.bank.to_string(), - mint_id: bank_mint_id, - ..Default::default() - } - ); + let bank_id = if let Some(id) = bank_data.id { + id + } else { + insert!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ) + }; let withdraw_event = WithdrawEvents { timestamp, @@ -603,73 +698,94 @@ impl MarginfiEvent for WithdrawEmissionsEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + let authority_data = entity_store.get_or_fetch_user(&self.authority.to_string())?; + let account_data = entity_store.get_or_fetch_account(&self.account.to_string())?; let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; - let emission_mint_data = entity_store.get_or_fetch_mint(&self.emissions_mint.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let authority_id = insert_if_needed!( - connection, - users, - Users, - self.authority.to_string(), - Users { - address: self.authority.to_string(), - ..Default::default() - } - ); + let authority_id = if let Some(id) = authority_data.id { + id + } else { + insert!( + connection, + users, + Users, + self.authority.to_string(), + Users { + address: self.authority.to_string(), + ..Default::default() + } + ) + }; - let account_id = insert_if_needed!( - connection, - accounts, - Accounts, - self.account.to_string(), - Accounts { - address: self.account.to_string(), - user_id: authority_id, - ..Default::default() - } - ); + let account_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + accounts, + Accounts, + self.account.to_string(), + Accounts { + address: self.account.to_string(), + user_id: authority_id, + ..Default::default() + } + ) + }; - let bank_mint_id = insert_if_needed!( - connection, - mints, - Mints, - bank_data.mint.address.clone(), - Mints { - address: bank_data.mint.address.clone(), - symbol: bank_data.mint.symbol.clone(), - decimals: bank_data.mint.decimals, - ..Default::default() - } - ); + let bank_mint_id = if let Some(id) = account_data.id { + id + } else { + insert!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ) + }; - let emission_mint_id = insert_if_needed!( - connection, - mints, - Mints, - self.emissions_mint.to_string(), - Mints { - address: self.emissions_mint.to_string(), - symbol: emission_mint_data.symbol.clone(), - decimals: emission_mint_data.decimals, - ..Default::default() - } - ); + let emission_mint_id = if let Some(id) = emission_mint_data.id { + id + } else { + insert!( + connection, + mints, + Mints, + self.emissions_mint.to_string(), + Mints { + address: self.emissions_mint.to_string(), + symbol: emission_mint_data.symbol.clone(), + decimals: emission_mint_data.decimals, + ..Default::default() + } + ) + }; - let bank_id = insert_if_needed!( - connection, - banks, - Banks, - self.bank.to_string(), - Banks { - address: self.bank.to_string(), - mint_id: bank_mint_id, - ..Default::default() - } - ); + let bank_id = if let Some(id) = bank_data.id { + id + } else { + insert!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ) + }; let withdraw_emissions_event = WithdrawEmissionsEvents { timestamp, @@ -714,111 +830,148 @@ impl MarginfiEvent for LiquidateEvent { db_connection: &mut PgConnection, entity_store: &mut EntityStore, ) -> Result<(), IndexingError> { + // Need to fetch this on explicitly as authority of the liquidator account might have changed since this event + let liquidator_user_data = + entity_store.get_or_fetch_user(&self.liquidator_authority.to_string())?; let asset_bank_data = entity_store.get_or_fetch_bank(&self.asset_bank.to_string())?; - let liability_bank_data = entity_store.get_or_fetch_bank(&self.liability_bank.to_string())?; - + let liquidator_account_data = + entity_store.get_or_fetch_account(&self.liquidator_account.to_string())?; let liquidatee_account_data = entity_store.get_or_fetch_account(&self.liquidatee_account.to_string())?; db_connection .transaction(|connection: &mut PgConnection| { - let liquidator_user_id = insert_if_needed!( - connection, - users, - Users, - self.liquidator_authority.to_string(), - Users { - address: self.liquidator_authority.to_string(), - ..Default::default() - } - ); + let liquidator_user_id = if let Some(id) = liquidator_user_data.id { + id + } else { + insert!( + connection, + users, + Users, + self.liquidator_authority.to_string(), + Users { + address: self.liquidator_authority.to_string(), + ..Default::default() + } + ) + }; - let liquidatee_user_id = insert_if_needed!( - connection, - users, - Users, - liquidatee_account_data.authority.clone(), - Users { - address: liquidatee_account_data.authority.clone(), - ..Default::default() - } - ); + // Using the current liquidatee account authority as we do no know the authority at time of liquidation (not in the event data) + let current_liquidatee_user_id = + if let Some(id) = liquidatee_account_data.authority.id { + id + } else { + get_and_insert_if_needed!( + connection, + users, + Users, + liquidatee_account_data.authority.address.clone(), + Users { + address: liquidatee_account_data.authority.address.clone(), + ..Default::default() + } + ) + }; - let liquidator_account_id = insert_if_needed!( - connection, - accounts, - Accounts, - self.liquidator_account.to_string(), - Accounts { - address: self.liquidator_account.to_string(), - user_id: liquidator_user_id, - ..Default::default() - } - ); + let liquidator_account_id = if let Some(id) = liquidator_account_data.id { + id + } else { + get_and_insert_if_needed!( + connection, + accounts, + Accounts, + self.liquidator_account.to_string(), + Accounts { + address: self.liquidator_account.to_string(), + user_id: liquidator_user_id, + ..Default::default() + } + ) + }; - let liquidatee_account_id = insert_if_needed!( - connection, - accounts, - Accounts, - self.liquidatee_account.to_string(), - Accounts { - address: self.liquidatee_account.to_string(), - user_id: liquidatee_user_id, - ..Default::default() - } - ); + let liquidatee_account_id = if let Some(id) = liquidatee_account_data.id { + id + } else { + get_and_insert_if_needed!( + connection, + accounts, + Accounts, + self.liquidatee_account.to_string(), + Accounts { + address: self.liquidatee_account.to_string(), + user_id: current_liquidatee_user_id, + ..Default::default() + } + ) + }; - let asset_mint_id = insert_if_needed!( - connection, - mints, - Mints, - asset_bank_data.mint.address.clone(), - Mints { - address: asset_bank_data.mint.address.clone(), - symbol: asset_bank_data.mint.symbol.clone(), - decimals: asset_bank_data.mint.decimals, - ..Default::default() - } - ); + let asset_mint_id = if let Some(id) = asset_bank_data.mint.id { + id + } else { + get_and_insert_if_needed!( + connection, + mints, + Mints, + asset_bank_data.mint.address.clone(), + Mints { + address: asset_bank_data.mint.address.clone(), + symbol: asset_bank_data.mint.symbol.clone(), + decimals: asset_bank_data.mint.decimals, + ..Default::default() + } + ) + }; - let liability_mint_id = insert_if_needed!( - connection, - mints, - Mints, - liability_bank_data.mint.address.clone(), - Mints { - address: liability_bank_data.mint.address.clone(), - symbol: liability_bank_data.mint.symbol.clone(), - decimals: liability_bank_data.mint.decimals, - ..Default::default() - } - ); + let liability_mint_id = if let Some(id) = liability_bank_data.mint.id { + id + } else { + get_and_insert_if_needed!( + connection, + mints, + Mints, + liability_bank_data.mint.address.clone(), + Mints { + address: liability_bank_data.mint.address.clone(), + symbol: liability_bank_data.mint.symbol.clone(), + decimals: liability_bank_data.mint.decimals, + ..Default::default() + } + ) + }; - let asset_bank_id = insert_if_needed!( - connection, - banks, - Banks, - self.asset_bank.to_string(), - Banks { - address: self.asset_bank.to_string(), - mint_id: asset_mint_id, - ..Default::default() - } - ); + let asset_bank_id = if let Some(id) = asset_bank_data.id { + id + } else { + get_and_insert_if_needed!( + connection, + banks, + Banks, + self.asset_bank.to_string(), + Banks { + address: self.asset_bank.to_string(), + mint_id: asset_mint_id, + ..Default::default() + } + ) + }; - let liability_bank_id = insert_if_needed!( - connection, - banks, - Banks, - self.liability_bank.to_string(), - Banks { - address: self.liability_bank.to_string(), - mint_id: liability_mint_id, - ..Default::default() - } - ); + let liability_bank_id = if let Some(id) = liability_bank_data.id { + id + } else { + get_and_insert_if_needed!( + connection, + banks, + Banks, + self.liability_bank.to_string(), + Banks { + address: self.liability_bank.to_string(), + mint_id: liability_mint_id, + ..Default::default() + } + ) + }; let liquidate_event = LiquidateEvents { timestamp, From 3846533f1fa88d3ed40146698b47261d9fe2e439 Mon Sep 17 00:00:00 2001 From: losman0s <95379755+losman0s@users.noreply.github.com> Date: Fri, 29 Mar 2024 01:06:24 +0800 Subject: [PATCH 07/20] fix: wrong mint id --- observability/Cargo.lock | 126 +++++++++--------- .../crates/event_indexer/src/parser.rs | 8 +- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/observability/Cargo.lock b/observability/Cargo.lock index 7ff4ee93..c16faf65 100644 --- a/observability/Cargo.lock +++ b/observability/Cargo.lock @@ -537,18 +537,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] name = "async-trait" -version = "0.1.78" +version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "461abc97219de0eaaf81fe3ef974a540158f3d079c2ab200f891f1a2ef201e85" +checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -564,9 +564,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "autotools" @@ -644,9 +644,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.70" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95d8e92cac0961e91dbd517496b00f7e9b92363dbe6d42c3198268323798860c" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -922,7 +922,7 @@ checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -933,9 +933,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "caps" @@ -1254,7 +1254,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "strsim 0.10.0", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1265,7 +1265,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1361,7 +1361,7 @@ dependencies = [ "dsl_auto_type", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1369,7 +1369,7 @@ name = "diesel_table_macro_syntax" version = "0.1.0" source = "git+https://github.com/diesel-rs/diesel.git?rev=12429cb4eebfb50581c494e1a183e4faed65c47f#12429cb4eebfb50581c494e1a183e4faed65c47f" dependencies = [ - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1400,7 +1400,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1442,7 +1442,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1530,7 +1530,7 @@ checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1542,7 +1542,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1638,9 +1638,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "feature-probe" @@ -1782,7 +1782,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1891,7 +1891,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.2.5", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -2188,9 +2188,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2244,9 +2244,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" @@ -2609,7 +2609,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -2711,7 +2711,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -2723,7 +2723,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -2858,7 +2858,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.2.5", + "indexmap 2.2.6", ] [[package]] @@ -2878,7 +2878,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -2957,12 +2957,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" dependencies = [ "proc-macro2 1.0.79", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -3062,7 +3062,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.53", + "syn 2.0.55", "tempfile", "which", ] @@ -3077,7 +3077,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -3288,9 +3288,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -3329,9 +3329,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", @@ -3653,7 +3653,7 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -3721,7 +3721,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -3737,9 +3737,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ "itoa", "ryu", @@ -3777,7 +3777,7 @@ dependencies = [ "darling", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -4086,7 +4086,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "rustc_version", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -4462,7 +4462,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "rustversion", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -4744,7 +4744,7 @@ checksum = "07fd7858fc4ff8fb0e34090e41d7eb06a823e1057945c26d480bfc21d2338a93" dependencies = [ "quote 1.0.35", "spl-discriminator-syn", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -4756,7 +4756,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "sha2 0.10.8", - "syn 2.0.53", + "syn 2.0.55", "thiserror", ] @@ -4813,7 +4813,7 @@ dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", "sha2 0.10.8", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -5011,9 +5011,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.53" +version = "2.0.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" +checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", @@ -5112,7 +5112,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -5227,7 +5227,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -5313,7 +5313,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "toml_datetime", "winnow", ] @@ -5324,7 +5324,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "toml_datetime", "winnow", ] @@ -5370,7 +5370,7 @@ dependencies = [ "proc-macro2 1.0.79", "prost-build", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -5437,7 +5437,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -5737,7 +5737,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", "wasm-bindgen-shared", ] @@ -5771,7 +5771,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6151,7 +6151,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -6171,7 +6171,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2 1.0.79", "quote 1.0.35", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 91224f45..7caa1e77 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -400,7 +400,7 @@ impl MarginfiEvent for BorrowEvent { ) }; - let bank_mint_id = if let Some(id) = account_data.id { + let bank_mint_id = if let Some(id) = bank_data.mint.id { id } else { insert!( @@ -511,7 +511,7 @@ impl MarginfiEvent for RepayEvent { ) }; - let bank_mint_id = if let Some(id) = account_data.id { + let bank_mint_id = if let Some(id) = bank_data.mint.id { id } else { insert!( @@ -623,7 +623,7 @@ impl MarginfiEvent for WithdrawEvent { ) }; - let bank_mint_id = if let Some(id) = account_data.id { + let bank_mint_id = if let Some(id) = bank_data.mint.id { id } else { insert!( @@ -737,7 +737,7 @@ impl MarginfiEvent for WithdrawEmissionsEvent { ) }; - let bank_mint_id = if let Some(id) = account_data.id { + let bank_mint_id = if let Some(id) = bank_data.mint.id { id } else { insert!( From c8e08ef27eee7daa22a8206f55867c51ac779c58 Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Fri, 29 Mar 2024 13:22:28 +0800 Subject: [PATCH 08/20] chore: cleanup --- observability/Cargo.lock | 6 + observability/crates/event_indexer/Cargo.toml | 4 + .../crates/event_indexer/bin/inspect_tx.rs | 78 +++++++++ observability/crates/rpc_utils/Cargo.toml | 6 + .../crates/rpc_utils/src/conversion.rs | 149 ++++++++++++++++++ observability/crates/rpc_utils/src/lib.rs | 2 + observability/indexer/Cargo.toml | 67 ++++---- .../indexer/src/commands/create_table.rs | 7 +- observability/indexer/src/commands/mod.rs | 3 - observability/indexer/src/entrypoint.rs | 38 ++--- observability/indexer/src/utils/mod.rs | 45 +++++- .../indexer/src/utils/transactions_crawler.rs | 4 +- 12 files changed, 337 insertions(+), 72 deletions(-) create mode 100644 observability/crates/event_indexer/bin/inspect_tx.rs create mode 100644 observability/crates/rpc_utils/src/conversion.rs diff --git a/observability/Cargo.lock b/observability/Cargo.lock index c16faf65..c253dbfb 100644 --- a/observability/Cargo.lock +++ b/observability/Cargo.lock @@ -3459,10 +3459,16 @@ dependencies = [ name = "rpc_utils" version = "0.1.0" dependencies = [ + "anyhow", "backoff", + "base64 0.21.7", + "bs58 0.4.0", "futures", "solana-client", "solana-sdk", + "solana-transaction-status", + "yellowstone-grpc-client", + "yellowstone-grpc-proto", ] [[package]] diff --git a/observability/crates/event_indexer/Cargo.toml b/observability/crates/event_indexer/Cargo.toml index 3429e5a5..7e748d13 100644 --- a/observability/crates/event_indexer/Cargo.toml +++ b/observability/crates/event_indexer/Cargo.toml @@ -11,6 +11,10 @@ path = "bin/index_events.rs" name = "backfill-events" path = "bin/backfill_events.rs" +[[bin]] +name = "inspect-tx" +path = "bin/inspect_tx.rs" + [features] default = ["mainnet-beta"] mainnet-beta = ["marginfi/mainnet-beta"] diff --git a/observability/crates/event_indexer/bin/inspect_tx.rs b/observability/crates/event_indexer/bin/inspect_tx.rs new file mode 100644 index 00000000..d56fdc4e --- /dev/null +++ b/observability/crates/event_indexer/bin/inspect_tx.rs @@ -0,0 +1,78 @@ +use std::{panic, process, str::FromStr}; + +use dotenv::dotenv; +use envconfig::Envconfig; +use event_indexer::{error::IndexingError,parser::{MarginfiEventParser, MARGINFI_GROUP_ADDRESS}}; +use tracing_subscriber::{layer::SubscriberExt, EnvFilter}; +use chrono::Utc; +use rpc_utils::conversion::convert_encoded_ui_transaction; +use solana_client::{nonblocking::rpc_client::RpcClient, rpc_config::RpcTransactionConfig}; +use solana_sdk::{commitment_config::CommitmentConfig, signature::Signature}; +use solana_transaction_status::UiTransactionEncoding; +use std::env; + +#[derive(Envconfig, Debug, Clone)] +pub struct Config { + #[envconfig(from = "PRETTY_LOGS")] + pub pretty_logs: Option, +} + +#[tokio::main] +pub async fn main() -> Result<(), IndexingError> { + dotenv().ok(); + + let orig_hook = panic::take_hook(); + panic::set_hook(Box::new(move |panic_info| { + orig_hook(panic_info); + process::exit(1); + })); + + let config = Config::init_from_env().unwrap(); + + let pretty_logs = config.pretty_logs.unwrap_or(false); + + let filter = EnvFilter::from_default_env(); + let stackdriver = tracing_stackdriver::layer(); // writes to std::io::Stdout + let subscriber = tracing_subscriber::registry().with(filter); + if pretty_logs { + let subscriber = subscriber.with(tracing_subscriber::fmt::layer().compact()); + tracing::subscriber::set_global_default(subscriber).unwrap(); + } else { + let subscriber = subscriber.with(stackdriver); + tracing::subscriber::set_global_default(subscriber).unwrap(); + }; + + let args: Vec = env::args().collect(); + let tx_signature = args.get(1).expect("Missing transaction signature argument"); + + let rpc_client = + RpcClient::new("https://mrgn.rpcpool.com/c293bade994b3864b52c6bbbba4b".to_string()); + + let event_parser = MarginfiEventParser::new(marginfi::ID, MARGINFI_GROUP_ADDRESS); + + let signature = Signature::from_str(&tx_signature).unwrap(); + + let encoded_tx = rpc_client + .get_transaction_with_config( + &signature, + RpcTransactionConfig { + max_supported_transaction_version: Some(0), + encoding: Some(UiTransactionEncoding::Base64), + commitment: Some(CommitmentConfig::confirmed()), + }, + ) + .await.unwrap(); + + let versioned_tx_with_meta = convert_encoded_ui_transaction(encoded_tx.transaction).unwrap(); + + let events = + event_parser.extract_events(Utc::now().timestamp(), encoded_tx.slot, versioned_tx_with_meta); + + if events.is_empty() { + println!("No event detected"); + } else { + println!("Events detected: {:#?}", events); + } + + Ok(()) +} diff --git a/observability/crates/rpc_utils/Cargo.toml b/observability/crates/rpc_utils/Cargo.toml index 1eff081a..8d277e34 100644 --- a/observability/crates/rpc_utils/Cargo.toml +++ b/observability/crates/rpc_utils/Cargo.toml @@ -4,7 +4,13 @@ version = "0.1.0" edition = "2021" [dependencies] +anyhow = "1.0.62" backoff = { workspace = true } +base64 = "0.21.0" +bs58 = "0.4.0" futures = { workspace = true } solana-client = { workspace = true } solana-sdk = { workspace = true } +solana-transaction-status = { workspace = true } +yellowstone-grpc-client = { workspace = true } +yellowstone-grpc-proto = { workspace = true } diff --git a/observability/crates/rpc_utils/src/conversion.rs b/observability/crates/rpc_utils/src/conversion.rs new file mode 100644 index 00000000..65e7d631 --- /dev/null +++ b/observability/crates/rpc_utils/src/conversion.rs @@ -0,0 +1,149 @@ +use std::str::FromStr; + +use anyhow::{anyhow, bail, Result}; +use base64::{engine::general_purpose::STANDARD, Engine}; +use solana_sdk::{account::Account, instruction::CompiledInstruction, message::v0::LoadedAddresses, pubkey::Pubkey, transaction_context::TransactionReturnData}; +use solana_transaction_status::{EncodedTransactionWithStatusMeta, InnerInstruction, InnerInstructions, TransactionStatusMeta, TransactionTokenBalance, UiInnerInstructions, UiInstruction, UiLoadedAddresses, UiTransactionReturnData, UiTransactionStatusMeta, UiTransactionTokenBalance, VersionedTransactionWithStatusMeta}; + + +pub fn convert_encoded_ui_transaction( + encoded_tx: EncodedTransactionWithStatusMeta, +) -> anyhow::Result { + Ok(VersionedTransactionWithStatusMeta { + transaction: encoded_tx.transaction.decode().unwrap(), + meta: convert_meta(encoded_tx.meta.unwrap())?, + }) +} + +pub fn convert_meta(ui_meta: UiTransactionStatusMeta) -> anyhow::Result { + let inner_instructions: Option> = ui_meta.inner_instructions.into(); + let log_messages: Option> = ui_meta.log_messages.into(); + let pre_token_balances: Option> = + ui_meta.pre_token_balances.into(); + let post_token_balances: Option> = + ui_meta.post_token_balances.into(); + let rewards: Option<_> = ui_meta.rewards.into(); + let return_data: Option = ui_meta.return_data.into(); + let compute_units_consumed: Option<_> = ui_meta.compute_units_consumed.into(); + let loaded_addresses: Option = ui_meta.loaded_addresses.into(); + + Ok(TransactionStatusMeta { + status: match ui_meta.err { + Some(err) => Err(err), + None => Ok(()), + }, + fee: ui_meta.fee, + pre_balances: ui_meta.pre_balances, + post_balances: ui_meta.post_balances, + inner_instructions: inner_instructions + .map(|ixs| { + ixs.into_iter() + .map(|ix| convert_inner_instructions(ix)) + .collect::>>() + }) + .transpose()?, + log_messages, + pre_token_balances: pre_token_balances + .map(|balances| { + balances + .into_iter() + .map(|balance| convert_token_balance(balance)) + .collect::>>() + }) + .transpose()?, + post_token_balances: post_token_balances + .map(|balances| { + balances + .into_iter() + .map(|balance| convert_token_balance(balance)) + .collect::>>() + }) + .transpose()?, + rewards, + loaded_addresses: convert_loaded_addresses(loaded_addresses.unwrap())?, + return_data: return_data + .map(|data| convert_return_data(data)) + .transpose()?, + compute_units_consumed, + }) +} + +fn convert_loaded_addresses( + ui_loaded_addresses: UiLoadedAddresses, +) -> anyhow::Result { + Ok(LoadedAddresses { + writable: ui_loaded_addresses + .writable + .into_iter() + .map(|address| Ok(Pubkey::from_str(&address)?)) + .collect::>>()?, + readonly: ui_loaded_addresses + .readonly + .into_iter() + .map(|address| Ok(Pubkey::from_str(&address)?)) + .collect::>>()?, + }) +} + +fn convert_return_data( + ui_return_data: UiTransactionReturnData, +) -> anyhow::Result { + Ok(TransactionReturnData { + program_id: Pubkey::from_str(&ui_return_data.program_id)?, + data: STANDARD.decode(&ui_return_data.data.0)?, + }) +} + +fn convert_token_balance( + ui_balance: UiTransactionTokenBalance, +) -> anyhow::Result { + let owner: Option<_> = ui_balance.owner.into(); + let program_id: Option<_> = ui_balance.program_id.into(); + + Ok(TransactionTokenBalance { + owner: owner.ok_or(anyhow!("Owner is missing"))?, + program_id: program_id.ok_or(anyhow!("Program id is missing"))?, + account_index: ui_balance.account_index, + mint: ui_balance.mint, + ui_token_amount: ui_balance.ui_token_amount, + }) +} + +fn convert_inner_instructions( + ui_instructions: UiInnerInstructions, +) -> anyhow::Result { + let index: Option<_> = ui_instructions.index.into(); + + Ok(InnerInstructions { + index: index.ok_or(anyhow!("Index is missing"))?, + instructions: ui_instructions + .instructions + .iter() + .map(|ix| match ix { + UiInstruction::Parsed(_) => { + bail!("There should not be parsed instruction here") + } + UiInstruction::Compiled(instruction) => Ok(InnerInstruction { + instruction: CompiledInstruction { + program_id_index: instruction.program_id_index, + accounts: instruction.accounts.clone(), + data: bs58::decode(&instruction.data).into_vec()?, + }, + stack_height: instruction.stack_height, + }), + }) + .collect::>>()?, + }) +} + +pub fn convert_account( + account_update: yellowstone_grpc_proto::geyser::SubscribeUpdateAccountInfo, +) -> Result { + Ok(Account { + lamports: account_update.lamports, + data: account_update.data, + owner: Pubkey::try_from(account_update.owner).unwrap(), + executable: account_update.executable, + rent_epoch: account_update.rent_epoch, + }) +} diff --git a/observability/crates/rpc_utils/src/lib.rs b/observability/crates/rpc_utils/src/lib.rs index 0bab36d9..bd801f79 100644 --- a/observability/crates/rpc_utils/src/lib.rs +++ b/observability/crates/rpc_utils/src/lib.rs @@ -1,3 +1,5 @@ +pub mod conversion; + use std::{collections::HashMap, iter::zip, time::Duration}; use backoff::{future::retry, ExponentialBackoffBuilder}; diff --git a/observability/indexer/Cargo.toml b/observability/indexer/Cargo.toml index afbdcc03..199161d3 100644 --- a/observability/indexer/Cargo.toml +++ b/observability/indexer/Cargo.toml @@ -8,16 +8,19 @@ name = "mfi-index" path = "src/bin/main.rs" [features] +default = ["mainnet-beta"] mainnet-beta = ["marginfi/mainnet-beta"] [dependencies] +solana-account-decoder = { workspace = true } solana-client = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } -solana-account-decoder = { workspace = true } + anchor-client = { workspace = true } +anchor-lang = { workspace = true } pyth-sdk-solana = { workspace = true } spl-token = { workspace = true } @@ -27,54 +30,56 @@ marginfi = { path = "../../programs/marginfi", features = [ "client", ] } +anyhow = "1.0.62" +backoff = { version = "0.4.0", features = ["tokio"] } +base64 = "0.21.0" +bincode = "1.3.3" +bs58 = "0.4.0" +bytemuck = "1.13.1" +bytes = "1.3.0" +chrono = "0.4.23" +chrono-tz = "0.8.0" +clap = { version = "3.2.23", features = ["derive"] } +concurrent-queue = "2.0.0" +crossbeam = "0.8.2" +dotenv = "0.15.0" +envconfig = "0.10.0" +futures = "0.3.25" gcp-bigquery-client = "0.16.7" -google-cloud-default = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994", features = ["pubsub"] } google-cloud-auth = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994" } -google-cloud-pubsub = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994" } +google-cloud-default = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994", features = ["pubsub"] } google-cloud-gax = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994" } google-cloud-googleapis = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994", features = ["bytes", "pubsub"] } -yup-oauth2 = "8.3.0" -yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", rev = "a2cd1498ac64baa1017d4a4cdefbf46100215b4c" } -yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", rev = "a2cd1498ac64baa1017d4a4cdefbf46100215b4c" } +google-cloud-pubsub = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994" } fixed = "1.12.0" fixed-macro = "1.2.0" -dotenv = "0.15.0" -bytemuck = "1.13.1" -tracing = "0.1.36" -tracing-stackdriver = "0.6.1" -tracing-subscriber = { version = "0.3.15", features = ["env-filter", "fmt"] } -clap = { version = "3.2.23", features = ["derive"] } -envconfig = "0.10.0" itertools = "0.10.5" json = "0.12.4" -bincode = "1.3.3" +lazy_static = "1.4.0" +prost = "0.11.0" +prost-derive = "0.11.2" +rayon = "1.6" serde = "1.0.147" serde_json = "1.0.88" serde_yaml = "0.9.14" -concurrent-queue = "2.0.0" +static_assertions = "1.1.0" +thiserror = "1.0" +tokio = { version = "1.14.1", features = ["full"] } +tokio-stream = "0.1.12" tonic = { version = "0.8.3", features = [ "tls", "tls-roots", "tls-webpki-roots", ] } -bs58 = "0.4.0" -bytes = "1.3.0" -thiserror = "1.0" -prost = "0.11.0" -prost-derive = "0.11.2" -tokio = { version = "1.14.1", features = ["full"] } -tokio-stream = "0.1.12" -futures = "0.3.25" -lazy_static = "1.4.0" -chrono = "0.4.23" -base64 = "0.21.0" +tracing = "0.1.36" +tracing-stackdriver = "0.6.1" +tracing-subscriber = { version = "0.3.15", features = ["env-filter", "fmt"] } uuid = { version = "1.2.2", features = ["v4"] } -chrono-tz = "0.8.0" -backoff = { version = "0.4.0", features = ["tokio"] } -rayon = "1.6" -anyhow = "1.0.62" +yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", rev = "a2cd1498ac64baa1017d4a4cdefbf46100215b4c" } +yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", rev = "a2cd1498ac64baa1017d4a4cdefbf46100215b4c" } +yup-oauth2 = "8.3.0" [build-dependencies] anyhow = "1.0.58" -tonic-build = "0.8.2" protobuf-src = "1.1.0" +tonic-build = "0.8.2" diff --git a/observability/indexer/src/commands/create_table.rs b/observability/indexer/src/commands/create_table.rs index b3e41826..317b0843 100644 --- a/observability/indexer/src/commands/create_table.rs +++ b/observability/indexer/src/commands/create_table.rs @@ -8,14 +8,13 @@ use tracing::{info, warn}; use yup_oauth2::parse_service_account_key; use crate::utils::big_query::{ - ACCOUNT_SCHEMA, METRIC_LENDING_POOL_BANK_SCHEMA, METRIC_MARGINFI_ACCOUNT_SCHEMA, - METRIC_MARGINFI_GROUP_SCHEMA, NOT_FOUND_CODE, TRANSACTION_SCHEMA, + METRIC_LENDING_POOL_BANK_SCHEMA, METRIC_MARGINFI_ACCOUNT_SCHEMA, METRIC_MARGINFI_GROUP_SCHEMA, + NOT_FOUND_CODE, TRANSACTION_SCHEMA, }; #[derive(Debug)] pub enum TableType { Transaction, - Account, MetricMarginfiGroup, MetricLendingPoolBank, MetricMarginfiAccount, @@ -27,7 +26,6 @@ impl FromStr for TableType { fn from_str(s: &str) -> Result { match s { "transaction" => Ok(Self::Transaction), - "account" => Ok(Self::Account), "metric_group" => Ok(Self::MetricMarginfiGroup), "metric_bank" => Ok(Self::MetricLendingPoolBank), "metric_account" => Ok(Self::MetricMarginfiAccount), @@ -54,7 +52,6 @@ pub async fn create_table( let schema = match table_type { TableType::Transaction => TRANSACTION_SCHEMA.to_owned(), - TableType::Account => ACCOUNT_SCHEMA.to_owned(), TableType::MetricMarginfiGroup => METRIC_MARGINFI_GROUP_SCHEMA.to_owned(), TableType::MetricLendingPoolBank => METRIC_LENDING_POOL_BANK_SCHEMA.to_owned(), TableType::MetricMarginfiAccount => METRIC_MARGINFI_ACCOUNT_SCHEMA.to_owned(), diff --git a/observability/indexer/src/commands/mod.rs b/observability/indexer/src/commands/mod.rs index 7c423c76..85e915bc 100644 --- a/observability/indexer/src/commands/mod.rs +++ b/observability/indexer/src/commands/mod.rs @@ -1,5 +1,2 @@ -pub mod backfill; pub mod create_table; -pub mod index_accounts; -pub mod index_transactions; pub mod snapshot_accounts; diff --git a/observability/indexer/src/entrypoint.rs b/observability/indexer/src/entrypoint.rs index 3881b679..e012e344 100644 --- a/observability/indexer/src/entrypoint.rs +++ b/observability/indexer/src/entrypoint.rs @@ -1,10 +1,10 @@ use crate::commands::create_table::TableType; -use crate::commands::index_accounts::{index_accounts, IndexAccountsConfig}; +use crate::commands::inspect_tx::inspect_tx; use crate::commands::snapshot_accounts::{snapshot_accounts, SnapshotAccountsConfig}; use crate::commands::{ - backfill::{backfill, BackfillConfig}, + backfill_events::{backfill_events, BackfillEventsConfig}, create_table::create_table, - index_transactions::{index_transactions, IndexTransactionsConfig}, + index_events::{index_events, IndexEventsConfig}, }; use anyhow::Result; use clap::Parser; @@ -48,14 +48,10 @@ pub enum Command { #[clap(long)] table_description: Option, }, - Backfill, - IndexTransactions, - IndexAccounts, SnapshotAccounts, } -#[tokio::main] -pub async fn entry(opts: Opts) -> Result<()> { +pub fn entry(opts: Opts) -> Result<()> { let orig_hook = panic::take_hook(); panic::set_hook(Box::new(move |panic_info| { orig_hook(panic_info); @@ -75,6 +71,8 @@ pub async fn entry(opts: Opts) -> Result<()> { tracing::subscriber::set_global_default(subscriber).unwrap(); }; + let rt = tokio::runtime::Runtime::new().unwrap(); + match opts.command { Command::CreateTable { project_id, @@ -83,7 +81,7 @@ pub async fn entry(opts: Opts) -> Result<()> { table_id, table_friendly_name, table_description, - } => { + } => rt.block_on(async { create_table( project_id, dataset_id, @@ -93,30 +91,12 @@ pub async fn entry(opts: Opts) -> Result<()> { table_description, ) .await - } - Command::Backfill => { - let config = BackfillConfig::init_from_env().unwrap(); - debug!("Config -> {:#?}", &config.clone()); - - backfill(config).await - } - Command::IndexTransactions => { - let config = IndexTransactionsConfig::init_from_env().unwrap(); - debug!("Config -> {:#?}", &config.clone()); - - index_transactions(config).await - } - Command::IndexAccounts => { - let config = IndexAccountsConfig::init_from_env().unwrap(); - debug!("Config -> {:#?}", &config.clone()); - - index_accounts(config).await - } + }), Command::SnapshotAccounts => { let config = SnapshotAccountsConfig::init_from_env().unwrap(); debug!("Config -> {:#?}", &config.clone()); - snapshot_accounts(config).await + rt.block_on(async { snapshot_accounts(config).await }) } } } diff --git a/observability/indexer/src/utils/mod.rs b/observability/indexer/src/utils/mod.rs index 7f8874e6..a1aa420b 100644 --- a/observability/indexer/src/utils/mod.rs +++ b/observability/indexer/src/utils/mod.rs @@ -1,10 +1,15 @@ -use solana_sdk::{account::Account, pubkey::Pubkey}; +use std::time::Duration; + +use futures::{stream, StreamExt}; +use solana_client::{nonblocking::rpc_client::RpcClient, rpc_config::RpcTransactionConfig}; +use solana_sdk::{account::Account, signature::Signature}; +use solana_transaction_status::{EncodedConfirmedTransactionWithStatusMeta, UiTransactionEncoding}; +use backoff::{future::retry, ExponentialBackoffBuilder}; pub mod big_query; pub mod errors; pub mod marginfi_account_dup; pub mod metrics; -pub mod protos; pub mod snapshot; pub mod transactions_crawler; @@ -18,4 +23,40 @@ pub fn convert_account( executable: account_update.executable, rent_epoch: account_update.rent_epoch, }) + } + +pub async fn get_multiple_transactions( + rpc_client: &RpcClient, + signatures: &[Signature], +) -> Result, String> +{ + Ok(stream::iter(signatures) + .map(|signature| async move { + ( + *signature, + retry( + ExponentialBackoffBuilder::new() + .with_max_interval(Duration::from_secs(5)) + .build(), + || async { + Ok(rpc_client + .get_transaction_with_config( + signature, + RpcTransactionConfig { + max_supported_transaction_version: Some(0), + encoding: Some(UiTransactionEncoding::Base64), + ..Default::default() + }, + ) + .await?) + }, + ) + .await + .map_err(|_| "Failed to get transaction".to_string()) + .unwrap(), + ) + }) + .buffered(10) // Higher ingest if unordered, but no way to order txs in same slot a posteriori in that case + .collect::>() + .await) } diff --git a/observability/indexer/src/utils/transactions_crawler.rs b/observability/indexer/src/utils/transactions_crawler.rs index 95a7b1c1..b5cc4707 100644 --- a/observability/indexer/src/utils/transactions_crawler.rs +++ b/observability/indexer/src/utils/transactions_crawler.rs @@ -34,7 +34,6 @@ pub const DEFAULT_MAX_CONCURRENT_REQUESTS: usize = 10; #[derive(Debug, Clone)] pub struct TransactionsCrawlerConfig { pub rpc_endpoint: String, - pub signature_fetch_limit: usize, pub max_concurrent_requests: usize, pub max_pending_signatures: usize, pub monitor_interval: u64, @@ -45,7 +44,6 @@ impl TransactionsCrawlerConfig { pub fn new(targets: Vec) -> Self { Self { rpc_endpoint: DEFAULT_RPC_ENDPOINT.to_string(), - signature_fetch_limit: DEFAULT_SIGNATURE_FETCH_LIMIT, max_concurrent_requests: DEFAULT_MAX_CONCURRENT_REQUESTS, max_pending_signatures: DEFAULT_MAX_PENDING_SIGNATURES, monitor_interval: DEFAULT_MONITOR_INTERVAL, @@ -63,6 +61,7 @@ pub struct SlotMeta { #[derive(Debug)] pub struct TransactionData { pub indexing_address: Pubkey, + pub signature: Signature, pub transaction: EncodedConfirmedTransactionWithStatusMeta, } @@ -347,6 +346,7 @@ impl TransactionsCrawler { .unwrap() .push(TransactionData { indexing_address: signature_data.indexing_address, + signature: signature_data.signature, transaction, }) .unwrap(); From f0b3e2cd7e793387e27322db91de0e5a10e8de7a Mon Sep 17 00:00:00 2001 From: losman0s <95379755+losman0s@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:19:04 +0800 Subject: [PATCH 09/20] feat: wip price tracking --- .../2024-03-14-151141_initial_tables/up.sql | 7 ++ .../crates/event_indexer/src/parser.rs | 95 +++++++++++++++---- 2 files changed, 83 insertions(+), 19 deletions(-) diff --git a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql index 7e04c35d..b7a4af1e 100644 --- a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql +++ b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql @@ -73,6 +73,7 @@ CREATE TABLE "deposit_events"( "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, + "price" NUMERIC, "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -88,6 +89,7 @@ CREATE TABLE "borrow_events"( "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, + "price" NUMERIC, "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -103,6 +105,7 @@ CREATE TABLE "repay_events"( "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, + "price" NUMERIC, "all" BOOLEAN NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() @@ -119,6 +122,7 @@ CREATE TABLE "withdraw_events"( "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, + "price" NUMERIC, "all" BOOLEAN NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() @@ -136,6 +140,7 @@ CREATE TABLE "withdraw_emissions_events"( "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "emission_mint_id" INT4 NOT NULL REFERENCES "mints"("id"), "amount" NUMERIC NOT NULL, + "price" NUMERIC, "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -153,6 +158,8 @@ CREATE TABLE "liquidate_events"( "asset_bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "liability_bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "asset_amount" NUMERIC NOT NULL, + "asset_price" NUMERIC, + "liability_price" NUMERIC, "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 7caa1e77..04b80b38 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -3,7 +3,8 @@ use std::collections::HashMap; use anchor_lang::{AnchorDeserialize, Discriminator}; use chrono::NaiveDateTime; use diesel::{ - Connection, ExpressionMethods, PgConnection, QueryDsl, RunQueryDsl, SelectableHelper, + Connection, ExpressionMethods, OptionalExtension, PgConnection, QueryDsl, RunQueryDsl, + SelectableHelper, }; use enum_dispatch::enum_dispatch; use marginfi::instruction::{ @@ -26,7 +27,7 @@ use solana_sdk::{ use solana_transaction_status::{ InnerInstruction, InnerInstructions, VersionedTransactionWithStatusMeta, }; -use tracing::{error, warn}; +use tracing::{error, info, warn}; use crate::{ db::{models::*, schema::*}, @@ -135,9 +136,16 @@ impl MarginfiEvent for CreateAccountEvent { ..Default::default() }; - diesel::insert_into(create_account_events::table) + let id: Option = diesel::insert_into(create_account_events::table) .values(&create_account_event) - .execute(connection)?; + .on_conflict_do_nothing() + .returning(create_account_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } diesel::result::QueryResult::Ok(()) }) @@ -225,9 +233,16 @@ impl MarginfiEvent for AccountAuthorityTransferEvent { ..Default::default() }; - diesel::insert_into(transfer_account_authority_events::table) + let id: Option = diesel::insert_into(transfer_account_authority_events::table) .values(&account_authority_transfer_event) - .execute(connection)?; + .on_conflict_do_nothing() + .returning(transfer_account_authority_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } diesel::result::QueryResult::Ok(()) }) @@ -335,9 +350,16 @@ impl MarginfiEvent for DepositEvent { ..Default::default() }; - diesel::insert_into(deposit_events::table) + let id: Option = diesel::insert_into(deposit_events::table) .values(&deposit_event) - .execute(connection)?; + .on_conflict_do_nothing() + .returning(deposit_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } diesel::result::QueryResult::Ok(()) }) @@ -445,9 +467,16 @@ impl MarginfiEvent for BorrowEvent { ..Default::default() }; - diesel::insert_into(borrow_events::table) + let id: Option = diesel::insert_into(borrow_events::table) .values(&borrow_event) - .execute(connection)?; + .on_conflict_do_nothing() + .returning(borrow_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } diesel::result::QueryResult::Ok(()) }) @@ -557,9 +586,16 @@ impl MarginfiEvent for RepayEvent { ..Default::default() }; - diesel::insert_into(repay_events::table) + let id: Option = diesel::insert_into(repay_events::table) .values(&repay_event) - .execute(connection)?; + .on_conflict_do_nothing() + .returning(repay_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } diesel::result::QueryResult::Ok(()) }) @@ -669,9 +705,16 @@ impl MarginfiEvent for WithdrawEvent { ..Default::default() }; - diesel::insert_into(withdraw_events::table) - .values(&withdraw_event) - .execute(connection)?; + let id: Option = diesel::insert_into(withdraw_events::table) + .values(vec![withdraw_event]) + .on_conflict_do_nothing() + .returning(withdraw_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } diesel::result::QueryResult::Ok(()) }) @@ -800,9 +843,16 @@ impl MarginfiEvent for WithdrawEmissionsEvent { ..Default::default() }; - diesel::insert_into(withdraw_emissions_events::table) + let id: Option = diesel::insert_into(withdraw_emissions_events::table) .values(&withdraw_emissions_event) - .execute(connection)?; + .on_conflict_do_nothing() + .returning(withdraw_emissions_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } diesel::result::QueryResult::Ok(()) }) @@ -987,9 +1037,16 @@ impl MarginfiEvent for LiquidateEvent { ..Default::default() }; - diesel::insert_into(liquidate_events::table) + let id: Option = diesel::insert_into(liquidate_events::table) .values(&liquidate_event) - .execute(connection)?; + .on_conflict_do_nothing() + .returning(liquidate_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } diesel::result::QueryResult::Ok(()) }) From 4f19ffbf5f556e996cb004190247fc2459e8b8fc Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Sun, 31 Mar 2024 17:02:24 +0800 Subject: [PATCH 10/20] feat: add + configure bank event --- observability/Cargo.lock | 1 + observability/crates/event_indexer/Cargo.toml | 1 + .../crates/event_indexer/bin/inspect_tx.rs | 19 +- .../2024-03-14-151141_initial_tables/up.sql | 119 ++++- .../crates/event_indexer/src/db/models.rs | 70 +++ .../crates/event_indexer/src/db/schema.rs | 103 ++++ .../crates/event_indexer/src/indexer.rs | 34 +- .../crates/event_indexer/src/parser.rs | 453 +++++++++++++++--- programs/marginfi/src/state/marginfi_group.rs | 4 +- 9 files changed, 735 insertions(+), 69 deletions(-) diff --git a/observability/Cargo.lock b/observability/Cargo.lock index c253dbfb..8d997a5e 100644 --- a/observability/Cargo.lock +++ b/observability/Cargo.lock @@ -1613,6 +1613,7 @@ dependencies = [ "dotenv", "enum_dispatch", "envconfig", + "fixed", "futures", "marginfi", "rpc_utils", diff --git a/observability/crates/event_indexer/Cargo.toml b/observability/crates/event_indexer/Cargo.toml index 7e748d13..d10334f4 100644 --- a/observability/crates/event_indexer/Cargo.toml +++ b/observability/crates/event_indexer/Cargo.toml @@ -29,6 +29,7 @@ diesel = { workspace = true, features = ["postgres", "chrono", "numeric"] } dotenv = { workspace = true } enum_dispatch = { workspace = true } envconfig = { workspace = true } +fixed = "1.12.0" futures = { workspace = true } marginfi = { workspace = true } rpc_utils = { path = "../rpc_utils" } diff --git a/observability/crates/event_indexer/bin/inspect_tx.rs b/observability/crates/event_indexer/bin/inspect_tx.rs index d56fdc4e..4fd10edd 100644 --- a/observability/crates/event_indexer/bin/inspect_tx.rs +++ b/observability/crates/event_indexer/bin/inspect_tx.rs @@ -1,15 +1,18 @@ use std::{panic, process, str::FromStr}; +use chrono::Utc; use dotenv::dotenv; use envconfig::Envconfig; -use event_indexer::{error::IndexingError,parser::{MarginfiEventParser, MARGINFI_GROUP_ADDRESS}}; -use tracing_subscriber::{layer::SubscriberExt, EnvFilter}; -use chrono::Utc; +use event_indexer::{ + error::IndexingError, + parser::{MarginfiEventParser, MARGINFI_GROUP_ADDRESS}, +}; use rpc_utils::conversion::convert_encoded_ui_transaction; use solana_client::{nonblocking::rpc_client::RpcClient, rpc_config::RpcTransactionConfig}; use solana_sdk::{commitment_config::CommitmentConfig, signature::Signature}; use solana_transaction_status::UiTransactionEncoding; use std::env; +use tracing_subscriber::{layer::SubscriberExt, EnvFilter}; #[derive(Envconfig, Debug, Clone)] pub struct Config { @@ -61,12 +64,16 @@ pub async fn main() -> Result<(), IndexingError> { commitment: Some(CommitmentConfig::confirmed()), }, ) - .await.unwrap(); + .await + .unwrap(); let versioned_tx_with_meta = convert_encoded_ui_transaction(encoded_tx.transaction).unwrap(); - let events = - event_parser.extract_events(Utc::now().timestamp(), encoded_tx.slot, versioned_tx_with_meta); + let events = event_parser.extract_events( + Utc::now().timestamp(), + encoded_tx.slot, + versioned_tx_with_meta, + ); if events.is_empty() { println!("No event detected"); diff --git a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql index b7a4af1e..a47a02db 100644 --- a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql +++ b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql @@ -1,10 +1,42 @@ --- Your SQL goes here +-- Enums + +CREATE TABLE "bank_operational_state" ( + "id" SERIAL PRIMARY KEY, + "name" VARCHAR NOT NULL +); + +INSERT INTO "bank_operational_state" ("name") +VALUES ('Paused'), + ('Operational'), + ('ReduceOnly'); + +CREATE TABLE "oracle_setup" ( + "id" SERIAL PRIMARY KEY, + "name" VARCHAR NOT NULL +); + +INSERT INTO "oracle_setup" ("name") +VALUES ('None'), + ('PythEma'), + ('SwitchboardV2'); + +CREATE TABLE "risk_tier" ( + "id" SERIAL PRIMARY KEY, + "name" VARCHAR NOT NULL +); + +INSERT INTO "risk_tier" ("name") +VALUES ('Collateral'), + ('Isolated'); + +-- Entities CREATE TABLE "mints"( "id" SERIAL PRIMARY KEY, "address" VARCHAR NOT NULL UNIQUE, "symbol" VARCHAR NOT NULL, "decimals" SMALLINT NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -14,6 +46,7 @@ CREATE TABLE "banks"( "id" SERIAL PRIMARY KEY, "address" VARCHAR NOT NULL UNIQUE, "mint_id" INT4 NOT NULL REFERENCES "mints"("id"), + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -22,6 +55,7 @@ SELECT diesel_manage_updated_at('banks'); CREATE TABLE "users"( "id" SERIAL PRIMARY KEY, "address" VARCHAR NOT NULL UNIQUE, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -31,19 +65,24 @@ CREATE TABLE "accounts"( "id" SERIAL PRIMARY KEY, "address" VARCHAR NOT NULL UNIQUE, "user_id" INT4 NOT NULL REFERENCES "users"("id"), + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); SELECT diesel_manage_updated_at('accounts'); +-- Events + CREATE TABLE "create_account_events"( "id" SERIAL PRIMARY KEY, "timestamp" TIMESTAMP NOT NULL, "tx_sig" VARCHAR NOT NULL, "in_flashloan" BOOLEAN NOT NULL, "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -55,9 +94,11 @@ CREATE TABLE "transfer_account_authority_events"( "tx_sig" VARCHAR NOT NULL, "in_flashloan" BOOLEAN NOT NULL, "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "old_authority_id" INT4 NOT NULL REFERENCES "users"("id"), "new_authority_id" INT4 NOT NULL REFERENCES "users"("id"), + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -69,11 +110,13 @@ CREATE TABLE "deposit_events"( "tx_sig" VARCHAR NOT NULL, "in_flashloan" BOOLEAN NOT NULL, "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, "price" NUMERIC, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -85,11 +128,13 @@ CREATE TABLE "borrow_events"( "tx_sig" VARCHAR NOT NULL, "in_flashloan" BOOLEAN NOT NULL, "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, "price" NUMERIC, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -101,12 +146,14 @@ CREATE TABLE "repay_events"( "tx_sig" VARCHAR NOT NULL, "in_flashloan" BOOLEAN NOT NULL, "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, "price" NUMERIC, "all" BOOLEAN NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -118,12 +165,14 @@ CREATE TABLE "withdraw_events"( "tx_sig" VARCHAR NOT NULL, "in_flashloan" BOOLEAN NOT NULL, "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "amount" NUMERIC NOT NULL, "price" NUMERIC, "all" BOOLEAN NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -135,12 +184,14 @@ CREATE TABLE "withdraw_emissions_events"( "tx_sig" VARCHAR NOT NULL, "in_flashloan" BOOLEAN NOT NULL, "call_stack" VARCHAR NOT NULL, + "account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "authority_id" INT4 NOT NULL REFERENCES "users"("id"), "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), "emission_mint_id" INT4 NOT NULL REFERENCES "mints"("id"), "amount" NUMERIC NOT NULL, "price" NUMERIC, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -152,6 +203,7 @@ CREATE TABLE "liquidate_events"( "tx_sig" VARCHAR NOT NULL, "in_flashloan" BOOLEAN NOT NULL, "call_stack" VARCHAR NOT NULL, + "liquidator_account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "liquidatee_account_id" INT4 NOT NULL REFERENCES "accounts"("id"), "liquidator_user_id" INT4 NOT NULL REFERENCES "users"("id"), @@ -160,7 +212,72 @@ CREATE TABLE "liquidate_events"( "asset_amount" NUMERIC NOT NULL, "asset_price" NUMERIC, "liability_price" NUMERIC, + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() ); SELECT diesel_manage_updated_at('liquidate_events'); + +CREATE TABLE "create_bank_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + + "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), + "asset_weight_init" NUMERIC NOT NULL, + "asset_weight_maint" NUMERIC NOT NULL, + "liability_weight_init" NUMERIC NOT NULL, + "liability_weight_maint" NUMERIC NOT NULL, + "deposit_limit" NUMERIC NOT NULL, + "optimal_utilization_rate" NUMERIC NOT NULL, + "plateau_interest_rate" NUMERIC NOT NULL, + "max_interest_rate" NUMERIC NOT NULL, + "insurance_fee_fixed_apr" NUMERIC NOT NULL, + "insurance_ir_fee" NUMERIC NOT NULL, + "protocol_fixed_fee_apr" NUMERIC NOT NULL, + "protocol_ir_fee" NUMERIC NOT NULL, + "operational_state_id" INT4 NOT NULL REFERENCES "bank_operational_state"("id"), + "oracle_setup_id" INT4 NOT NULL REFERENCES "oracle_setup"("id"), + "oracle_keys" VARCHAR NOT NULL, + "borrow_limit" NUMERIC NOT NULL, + "risk_tier_id" INT4 NOT NULL REFERENCES "risk_tier"("id"), + "total_asset_value_init_limit" NUMERIC NOT NULL, + + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() +); +SELECT diesel_manage_updated_at('create_bank_events'); + +CREATE TABLE "configure_bank_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + + "bank_id" INT4 NOT NULL REFERENCES "banks"("id"), + "asset_weight_init" NUMERIC, + "asset_weight_maint" NUMERIC, + "liability_weight_init" NUMERIC, + "liability_weight_maint" NUMERIC, + "deposit_limit" NUMERIC, + "borrow_limit" NUMERIC, + "operational_state_id" INT4 REFERENCES "bank_operational_state"("id"), + "oracle_setup_id" INT4 REFERENCES "oracle_setup"("id"), + "oracle_keys" VARCHAR, + "optimal_utilization_rate" NUMERIC, + "plateau_interest_rate" NUMERIC, + "max_interest_rate" NUMERIC, + "insurance_fee_fixed_apr" NUMERIC, + "insurance_ir_fee" NUMERIC, + "protocol_fixed_fee_apr" NUMERIC, + "protocol_ir_fee" NUMERIC, + "risk_tier_id" INT4 REFERENCES "risk_tier"("id"), + "total_asset_value_init_limit" NUMERIC, + + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() +); +SELECT diesel_manage_updated_at('configure_bank_events'); diff --git a/observability/crates/event_indexer/src/db/models.rs b/observability/crates/event_indexer/src/db/models.rs index 92fc77af..29877ec8 100644 --- a/observability/crates/event_indexer/src/db/models.rs +++ b/observability/crates/event_indexer/src/db/models.rs @@ -50,6 +50,7 @@ pub struct CreateAccountEvents { pub tx_sig: String, pub in_flashloan: bool, pub call_stack: String, + pub account_id: i32, pub authority_id: i32, } @@ -63,6 +64,7 @@ pub struct TransferAccountAuthorityEvents { pub tx_sig: String, pub in_flashloan: bool, pub call_stack: String, + pub account_id: i32, pub old_authority_id: i32, pub new_authority_id: i32, @@ -77,6 +79,7 @@ pub struct DepositEvents { pub tx_sig: String, pub in_flashloan: bool, pub call_stack: String, + pub account_id: i32, pub authority_id: i32, pub bank_id: i32, @@ -92,6 +95,7 @@ pub struct BorrowEvents { pub tx_sig: String, pub in_flashloan: bool, pub call_stack: String, + pub account_id: i32, pub authority_id: i32, pub bank_id: i32, @@ -107,6 +111,7 @@ pub struct RepayEvents { pub tx_sig: String, pub in_flashloan: bool, pub call_stack: String, + pub account_id: i32, pub authority_id: i32, pub bank_id: i32, @@ -123,6 +128,7 @@ pub struct WithdrawEvents { pub tx_sig: String, pub in_flashloan: bool, pub call_stack: String, + pub account_id: i32, pub authority_id: i32, pub bank_id: i32, @@ -139,6 +145,7 @@ pub struct WithdrawEmissionsEvents { pub tx_sig: String, pub in_flashloan: bool, pub call_stack: String, + pub account_id: i32, pub authority_id: i32, pub bank_id: i32, @@ -155,6 +162,7 @@ pub struct LiquidateEvents { pub tx_sig: String, pub in_flashloan: bool, pub call_stack: String, + pub liquidator_account_id: i32, pub liquidatee_account_id: i32, pub liquidator_user_id: i32, @@ -162,3 +170,65 @@ pub struct LiquidateEvents { pub liability_bank_id: i32, pub asset_amount: Decimal, } + +#[derive(Default, Debug, Queryable, Selectable, Insertable)] +#[diesel(table_name = create_bank_events)] +pub struct CreateBankEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + + pub bank_id: i32, + pub asset_weight_init: Decimal, + pub asset_weight_maint: Decimal, + pub liability_weight_init: Decimal, + pub liability_weight_maint: Decimal, + pub deposit_limit: Decimal, + pub optimal_utilization_rate: Decimal, + pub plateau_interest_rate: Decimal, + pub max_interest_rate: Decimal, + pub insurance_fee_fixed_apr: Decimal, + pub insurance_ir_fee: Decimal, + pub protocol_fixed_fee_apr: Decimal, + pub protocol_ir_fee: Decimal, + pub operational_state_id: i32, + pub oracle_setup_id: i32, + pub oracle_keys: String, + pub borrow_limit: Decimal, + pub risk_tier_id: i32, + pub total_asset_value_init_limit: Decimal, +} + +#[derive(Default, Debug, Queryable, Selectable, Insertable)] +#[diesel(table_name = configure_bank_events)] +pub struct ConfigureBankEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, + + pub bank_id: i32, + pub asset_weight_init: Option, + pub asset_weight_maint: Option, + pub liability_weight_init: Option, + pub liability_weight_maint: Option, + pub deposit_limit: Option, + pub optimal_utilization_rate: Option, + pub plateau_interest_rate: Option, + pub max_interest_rate: Option, + pub insurance_fee_fixed_apr: Option, + pub insurance_ir_fee: Option, + pub protocol_fixed_fee_apr: Option, + pub protocol_ir_fee: Option, + pub operational_state_id: Option, + pub oracle_setup_id: Option, + pub oracle_keys: Option, + pub borrow_limit: Option, + pub risk_tier_id: Option, + pub total_asset_value_init_limit: Option, +} diff --git a/observability/crates/event_indexer/src/db/schema.rs b/observability/crates/event_indexer/src/db/schema.rs index 45c30186..de5d41d4 100644 --- a/observability/crates/event_indexer/src/db/schema.rs +++ b/observability/crates/event_indexer/src/db/schema.rs @@ -10,6 +10,13 @@ diesel::table! { } } +diesel::table! { + bank_operational_state (id) { + id -> Int4, + name -> Varchar, + } +} + diesel::table! { banks (id) { id -> Int4, @@ -31,6 +38,38 @@ diesel::table! { authority_id -> Int4, bank_id -> Int4, amount -> Numeric, + price -> Nullable, + created_at -> Timestamp, + updated_at -> Timestamp, + } +} + +diesel::table! { + configure_bank_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + bank_id -> Int4, + asset_weight_init -> Nullable, + asset_weight_maint -> Nullable, + liability_weight_init -> Nullable, + liability_weight_maint -> Nullable, + deposit_limit -> Nullable, + borrow_limit -> Nullable, + operational_state_id -> Nullable, + oracle_setup_id -> Nullable, + oracle_keys -> Nullable, + optimal_utilization_rate -> Nullable, + plateau_interest_rate -> Nullable, + max_interest_rate -> Nullable, + insurance_fee_fixed_apr -> Nullable, + insurance_ir_fee -> Nullable, + protocol_fixed_fee_apr -> Nullable, + protocol_ir_fee -> Nullable, + risk_tier_id -> Nullable, + total_asset_value_init_limit -> Nullable, created_at -> Timestamp, updated_at -> Timestamp, } @@ -50,6 +89,37 @@ diesel::table! { } } +diesel::table! { + create_bank_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + bank_id -> Int4, + asset_weight_init -> Numeric, + asset_weight_maint -> Numeric, + liability_weight_init -> Numeric, + liability_weight_maint -> Numeric, + deposit_limit -> Numeric, + optimal_utilization_rate -> Numeric, + plateau_interest_rate -> Numeric, + max_interest_rate -> Numeric, + insurance_fee_fixed_apr -> Numeric, + insurance_ir_fee -> Numeric, + protocol_fixed_fee_apr -> Numeric, + protocol_ir_fee -> Numeric, + operational_state_id -> Int4, + oracle_setup_id -> Int4, + oracle_keys -> Varchar, + borrow_limit -> Numeric, + risk_tier_id -> Int4, + total_asset_value_init_limit -> Numeric, + created_at -> Timestamp, + updated_at -> Timestamp, + } +} + diesel::table! { deposit_events (id) { id -> Int4, @@ -61,6 +131,7 @@ diesel::table! { authority_id -> Int4, bank_id -> Int4, amount -> Numeric, + price -> Nullable, created_at -> Timestamp, updated_at -> Timestamp, } @@ -79,6 +150,8 @@ diesel::table! { asset_bank_id -> Int4, liability_bank_id -> Int4, asset_amount -> Numeric, + asset_price -> Nullable, + liability_price -> Nullable, created_at -> Timestamp, updated_at -> Timestamp, } @@ -95,6 +168,13 @@ diesel::table! { } } +diesel::table! { + oracle_setup (id) { + id -> Int4, + name -> Varchar, + } +} + diesel::table! { repay_events (id) { id -> Int4, @@ -106,12 +186,20 @@ diesel::table! { authority_id -> Int4, bank_id -> Int4, amount -> Numeric, + price -> Nullable, all -> Bool, created_at -> Timestamp, updated_at -> Timestamp, } } +diesel::table! { + risk_tier (id) { + id -> Int4, + name -> Varchar, + } +} + diesel::table! { transfer_account_authority_events (id) { id -> Int4, @@ -148,6 +236,7 @@ diesel::table! { bank_id -> Int4, emission_mint_id -> Int4, amount -> Numeric, + price -> Nullable, created_at -> Timestamp, updated_at -> Timestamp, } @@ -164,6 +253,7 @@ diesel::table! { authority_id -> Int4, bank_id -> Int4, amount -> Numeric, + price -> Nullable, all -> Bool, created_at -> Timestamp, updated_at -> Timestamp, @@ -175,8 +265,16 @@ diesel::joinable!(banks -> mints (mint_id)); diesel::joinable!(borrow_events -> accounts (account_id)); diesel::joinable!(borrow_events -> banks (bank_id)); diesel::joinable!(borrow_events -> users (authority_id)); +diesel::joinable!(configure_bank_events -> bank_operational_state (operational_state_id)); +diesel::joinable!(configure_bank_events -> banks (bank_id)); +diesel::joinable!(configure_bank_events -> oracle_setup (oracle_setup_id)); +diesel::joinable!(configure_bank_events -> risk_tier (risk_tier_id)); diesel::joinable!(create_account_events -> accounts (account_id)); diesel::joinable!(create_account_events -> users (authority_id)); +diesel::joinable!(create_bank_events -> bank_operational_state (operational_state_id)); +diesel::joinable!(create_bank_events -> banks (bank_id)); +diesel::joinable!(create_bank_events -> oracle_setup (oracle_setup_id)); +diesel::joinable!(create_bank_events -> risk_tier (risk_tier_id)); diesel::joinable!(deposit_events -> accounts (account_id)); diesel::joinable!(deposit_events -> banks (bank_id)); diesel::joinable!(deposit_events -> users (authority_id)); @@ -195,13 +293,18 @@ diesel::joinable!(withdraw_events -> users (authority_id)); diesel::allow_tables_to_appear_in_same_query!( accounts, + bank_operational_state, banks, borrow_events, + configure_bank_events, create_account_events, + create_bank_events, deposit_events, liquidate_events, mints, + oracle_setup, repay_events, + risk_tier, transfer_account_authority_events, users, withdraw_emissions_events, diff --git a/observability/crates/event_indexer/src/indexer.rs b/observability/crates/event_indexer/src/indexer.rs index 6fa718df..cf2421b2 100644 --- a/observability/crates/event_indexer/src/indexer.rs +++ b/observability/crates/event_indexer/src/indexer.rs @@ -5,6 +5,7 @@ use std::{ vec, }; +use backoff::{exponential::ExponentialBackoffBuilder, retry, SystemClock}; use chrono::DateTime; use crossbeam::channel::{Receiver, Sender}; use diesel::PgConnection; @@ -297,16 +298,37 @@ async fn store_events( ) .unwrap_or_else(|_| "null".to_string()); println!("Storing event: {:?}", event); - event - .db_insert( + + let mut retries = 0; + retry( + ExponentialBackoffBuilder::::new() + .with_max_interval(Duration::from_secs(5)) + .build(), + || match event.db_insert( timestamp, - tx_sig, + tx_sig.clone(), in_flashloan, - call_stack, + call_stack.clone(), db_connection, entity_store, - ) - .unwrap(); + ) { + Ok(signatures) => Ok(signatures), + Err(e) => { + if retries > 5 { + error!( + "Failed to insert event after 5 retries: {:?} - {:?}", + event, e + ); + Err(backoff::Error::permanent(e)) + } else { + warn!("Failed to insert event, retrying: {:?}", e); + retries += 1; + Err(backoff::Error::transient(e)) + } + } + }, + ) + .unwrap(); } } } diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 04b80b38..997bd593 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -7,12 +7,17 @@ use diesel::{ SelectableHelper, }; use enum_dispatch::enum_dispatch; -use marginfi::instruction::{ - LendingAccountBorrow, LendingAccountCloseBalance, LendingAccountDeposit, - LendingAccountEndFlashloan, LendingAccountLiquidate, LendingAccountRepay, - LendingAccountSettleEmissions, LendingAccountStartFlashloan, LendingAccountWithdraw, - LendingAccountWithdrawEmissions, LendingPoolAccrueBankInterest, LendingPoolAddBankWithSeed, - LendingPoolConfigureBank, MarginfiAccountInitialize, SetNewAccountAuthority, +use fixed::types::I80F48; +use marginfi::{ + instruction::{ + LendingAccountBorrow, LendingAccountCloseBalance, LendingAccountDeposit, + LendingAccountEndFlashloan, LendingAccountLiquidate, LendingAccountRepay, + LendingAccountSettleEmissions, LendingAccountStartFlashloan, LendingAccountWithdraw, + LendingAccountWithdrawEmissions, LendingPoolAccrueBankInterest, LendingPoolAddBank, + LendingPoolAddBankWithSeed, LendingPoolConfigureBank, MarginfiAccountInitialize, + SetNewAccountAuthority, + }, + state::marginfi_group::{BankConfig, BankConfigCompact, BankConfigOpt}, }; use rust_decimal::{prelude::FromPrimitive, Decimal}; use solana_sdk::{ @@ -38,7 +43,7 @@ use crate::{ const SPL_TRANSFER_DISCRIMINATOR: u8 = 3; pub const MARGINFI_GROUP_ADDRESS: Pubkey = pubkey!("4qp6Fx6tnZkY5Wropq9wUYgtFxXKwE6viZxFHg3rdAG8"); -const COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT: u64 = 232_836_972; +const COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT: u64 = 232_933_019; #[derive(Debug)] pub struct MarginfiEventWithMeta { @@ -74,8 +79,10 @@ pub enum Event { Withdraw(WithdrawEvent), WithdrawEmissions(WithdrawEmissionsEvent), Liquidate(LiquidateEvent), - // // Admin actions - // AddBank(AddBankEvent), + + // Admin actions + AddBank(AddBankEvent), + ConfigureBank(ConfigureBankEvent), } #[derive(Debug)] @@ -1054,25 +1061,330 @@ impl MarginfiEvent for LiquidateEvent { } } -// #[derive(Debug)] -// pub struct AddBankEvent { -// pub bank: Pubkey, -// pub mint: Pubkey, -// pub config: BankConfig, -// } - -// impl MarginfiEvent for AddBankEvent { -// fn db_insert( -// &self, -// timestamp: NaiveDateTime, -// tx_sig: String, -// in_flashloan: bool, -// call_stack: String, -// db_connection: &mut PgConnection, -// ) -> Result<(), IndexingError> { -// todo!("AddBankEvent::db_insert") -// } -// } +#[derive(Debug)] +pub struct AddBankEvent { + pub bank: Pubkey, + pub mint: Pubkey, + pub config: BankConfig, +} + +impl MarginfiEvent for AddBankEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + let mint_data = entity_store.get_or_fetch_mint(&self.mint.to_string())?; + + db_connection + .transaction(|connection: &mut PgConnection| { + let bank_mint_id = if let Some(id) = mint_data.id { + id + } else { + insert!( + connection, + mints, + Mints, + self.mint.to_string(), + Mints { + address: mint_data.address.clone(), + symbol: mint_data.symbol.clone(), + decimals: mint_data.decimals, + ..Default::default() + } + ) + }; + + // Not RPC fetching the account data here because it could lead to race condition with the RPC when live ingesting, + let bank_id = get_and_insert_if_needed!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ); + + let create_bank_event = CreateBankEvents { + timestamp, + tx_sig, + call_stack, + in_flashloan, + + bank_id, + asset_weight_init: Decimal::from_f64( + I80F48::from(self.config.asset_weight_init).to_num(), + ) + .unwrap(), + asset_weight_maint: Decimal::from_f64( + I80F48::from(self.config.asset_weight_maint).to_num(), + ) + .unwrap(), + liability_weight_init: Decimal::from_f64( + I80F48::from(self.config.liability_weight_init).to_num(), + ) + .unwrap(), + liability_weight_maint: Decimal::from_f64( + I80F48::from(self.config.liability_weight_maint).to_num(), + ) + .unwrap(), + deposit_limit: Decimal::from_u64(self.config.deposit_limit).unwrap(), + optimal_utilization_rate: Decimal::from_f64( + I80F48::from(self.config.interest_rate_config.optimal_utilization_rate) + .to_num(), + ) + .unwrap(), + plateau_interest_rate: Decimal::from_f64( + I80F48::from(self.config.interest_rate_config.plateau_interest_rate) + .to_num(), + ) + .unwrap(), + max_interest_rate: Decimal::from_f64( + I80F48::from(self.config.interest_rate_config.max_interest_rate).to_num(), + ) + .unwrap(), + insurance_fee_fixed_apr: Decimal::from_f64( + I80F48::from(self.config.interest_rate_config.insurance_fee_fixed_apr) + .to_num(), + ) + .unwrap(), + insurance_ir_fee: Decimal::from_f64( + I80F48::from(self.config.interest_rate_config.insurance_ir_fee).to_num(), + ) + .unwrap(), + protocol_fixed_fee_apr: Decimal::from_f64( + I80F48::from(self.config.interest_rate_config.protocol_fixed_fee_apr) + .to_num(), + ) + .unwrap(), + protocol_ir_fee: Decimal::from_f64( + I80F48::from(self.config.interest_rate_config.protocol_ir_fee).to_num(), + ) + .unwrap(), + operational_state_id: self.config.operational_state as i32, + oracle_setup_id: self.config.oracle_setup as i32, + oracle_keys: serde_json::to_string( + &self + .config + .oracle_keys + .iter() + .map(|k| k.to_string()) + .collect::>(), + ) + .unwrap(), + borrow_limit: Decimal::from_u64(self.config.borrow_limit).unwrap(), + risk_tier_id: self.config.risk_tier as i32, + + ..Default::default() + }; + + let id: Option = diesel::insert_into(create_bank_events::table) + .values(&create_bank_event) + .on_conflict_do_nothing() + .returning(create_bank_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } +} + +#[derive(Debug)] +pub struct ConfigureBankEvent { + pub bank: Pubkey, + pub config: BankConfigOpt, +} + +impl MarginfiEvent for ConfigureBankEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + let bank_data = entity_store.get_or_fetch_bank(&self.bank.to_string())?; + + db_connection + .transaction(|connection: &mut PgConnection| { + let bank_mint_id = if let Some(id) = bank_data.mint.id { + id + } else { + insert!( + connection, + mints, + Mints, + bank_data.mint.address.clone(), + Mints { + address: bank_data.mint.address.clone(), + symbol: bank_data.mint.symbol.clone(), + decimals: bank_data.mint.decimals, + ..Default::default() + } + ) + }; + + let bank_id = if let Some(id) = bank_data.id { + id + } else { + insert!( + connection, + banks, + Banks, + self.bank.to_string(), + Banks { + address: self.bank.to_string(), + mint_id: bank_mint_id, + ..Default::default() + } + ) + }; + + let configure_bank_event = ConfigureBankEvents { + timestamp, + tx_sig, + call_stack, + in_flashloan, + + bank_id, + asset_weight_init: self + .config + .asset_weight_init + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()), + asset_weight_maint: self + .config + .asset_weight_maint + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()), + liability_weight_init: self + .config + .liability_weight_init + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()), + liability_weight_maint: self + .config + .liability_weight_maint + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()), + deposit_limit: self + .config + .deposit_limit + .map(|v| Decimal::from_u64(v).unwrap()), + optimal_utilization_rate: self + .config + .interest_rate_config + .as_ref() + .map(|v| { + v.optimal_utilization_rate + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()) + }) + .flatten() + .clone(), + plateau_interest_rate: self + .config + .interest_rate_config + .as_ref() + .map(|v| { + v.plateau_interest_rate + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()) + }) + .flatten() + .clone(), + max_interest_rate: self + .config + .interest_rate_config + .as_ref() + .map(|v| { + v.max_interest_rate + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()) + }) + .flatten() + .clone(), + insurance_fee_fixed_apr: self + .config + .interest_rate_config + .as_ref() + .map(|v| { + v.insurance_fee_fixed_apr + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()) + }) + .flatten() + .clone(), + insurance_ir_fee: self + .config + .interest_rate_config + .as_ref() + .map(|v| { + v.insurance_ir_fee + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()) + }) + .flatten() + .clone(), + protocol_fixed_fee_apr: self + .config + .interest_rate_config + .as_ref() + .map(|v| { + v.protocol_fixed_fee_apr + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()) + }) + .flatten() + .clone(), + protocol_ir_fee: self + .config + .interest_rate_config + .as_ref() + .map(|v| { + v.protocol_ir_fee + .map(|v| Decimal::from_f64(I80F48::from(v).to_num()).unwrap()) + }) + .flatten() + .clone(), + operational_state_id: self.config.operational_state.map(|v| v as i32), + oracle_setup_id: self.config.oracle.map(|v| v.setup as i32), + oracle_keys: self.config.oracle.map(|v| { + serde_json::to_string( + &v.keys.iter().map(|k| k.to_string()).collect::>(), + ) + .unwrap() + }), + borrow_limit: self + .config + .borrow_limit + .map(|v| Decimal::from_u64(v).unwrap()), + risk_tier_id: self.config.risk_tier.map(|v| v as i32), + + ..Default::default() + }; + + let id: Option = diesel::insert_into(configure_bank_events::table) + .values(&configure_bank_event) + .on_conflict_do_nothing() + .returning(configure_bank_events::id) + .get_result(connection) + .optional()?; + + if id.is_none() { + info!("event already exists"); + } + + diesel::result::QueryResult::Ok(()) + }) + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string())) + } +} pub struct MarginfiEventParser { program_id: Pubkey, @@ -1437,29 +1749,64 @@ impl MarginfiEventParser { amount: spl_transfer_amount, })) } - // LendingPoolAddBank::DISCRIMINATOR => { - // let marginfi_group = *ix_accounts.get(0).unwrap(); - // if !marginfi_group.eq(&self.marginfi_group) { - // return None; - // } - - // let bank_config = if slot < COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT { - // BankConfig::deserialize(&mut &instruction_data[..531]).unwrap() - // } else { - // BankConfigCompact::deserialize(&mut &instruction_data[..531]) - // .unwrap() - // .into() - // }; - - // let bank_mint = *ix_accounts.get(3).unwrap(); - // let bank = *ix_accounts.get(4).unwrap(); - - // Some(Event::AddBank(AddBankEvent { - // bank, - // mint: bank_mint, - // config: bank_config, - // })) - // } + LendingPoolAddBank::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + let bank_config = if slot < COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT { + BankConfig::deserialize(&mut &instruction_data[..531]).unwrap() + } else { + BankConfigCompact::deserialize(&mut &instruction_data[..363]) + .unwrap() + .into() + }; + + let bank_mint = *ix_accounts.get(3).unwrap(); + let bank = *ix_accounts.get(4).unwrap(); + + Some(Event::AddBank(AddBankEvent { + bank, + mint: bank_mint, + config: bank_config, + })) + } + LendingPoolAddBankWithSeed::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + let bank_config = BankConfigCompact::deserialize(&mut &instruction_data[..363]) + .unwrap() + .into(); + + let bank_mint = *ix_accounts.get(3).unwrap(); + let bank = *ix_accounts.get(4).unwrap(); + + Some(Event::AddBank(AddBankEvent { + bank, + mint: bank_mint, + config: bank_config, + })) + } + LendingPoolConfigureBank::DISCRIMINATOR => { + let marginfi_group = *ix_accounts.get(0).unwrap(); + if !marginfi_group.eq(&self.marginfi_group) { + return None; + } + + let bank_config_opt = + BankConfigOpt::deserialize(&mut &instruction_data[..363]).unwrap(); + + let bank = *ix_accounts.get(2).unwrap(); + + Some(Event::ConfigureBank(ConfigureBankEvent { + bank, + config: bank_config_opt, + })) + } LendingAccountStartFlashloan::DISCRIMINATOR => { *in_flashloan = true; @@ -1472,9 +1819,7 @@ impl MarginfiEventParser { } LendingAccountCloseBalance::DISCRIMINATOR | LendingPoolAccrueBankInterest::DISCRIMINATOR - | LendingAccountSettleEmissions::DISCRIMINATOR - | LendingPoolConfigureBank::DISCRIMINATOR - | LendingPoolAddBankWithSeed::DISCRIMINATOR => None, + | LendingAccountSettleEmissions::DISCRIMINATOR => None, _ => { warn!( "Unknown instruction discriminator {:?} in {:?}", diff --git a/programs/marginfi/src/state/marginfi_group.rs b/programs/marginfi/src/state/marginfi_group.rs index 4c9167d8..cc1885a8 100644 --- a/programs/marginfi/src/state/marginfi_group.rs +++ b/programs/marginfi/src/state/marginfi_group.rs @@ -1125,7 +1125,7 @@ impl From for I80F48 { #[cfg_attr( any(feature = "test", feature = "client"), - derive(Clone, PartialEq, Eq, TypeLayout) + derive(Debug, Clone, PartialEq, Eq, TypeLayout) )] #[derive(AnchorDeserialize, AnchorSerialize, Default)] pub struct BankConfigOpt { @@ -1151,7 +1151,7 @@ pub struct BankConfigOpt { #[cfg_attr( any(feature = "test", feature = "client"), - derive(PartialEq, Eq, TypeLayout) + derive(Debug, PartialEq, Eq, TypeLayout) )] #[derive(Clone, Copy, AnchorDeserialize, AnchorSerialize)] pub struct OracleConfig { From b783c5b29bf106dbe067443c52e270fb8d43f586 Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Wed, 3 Apr 2024 00:00:46 +0800 Subject: [PATCH 11/20] feat: range-based crawling --- observability/Cargo.lock | 1 + observability/crates/event_indexer/Cargo.toml | 1 + .../event_indexer/bin/backfill_events.rs | 255 ++++++++++++------ .../2024-03-14-151141_initial_tables/up.sql | 22 +- .../crates/event_indexer/src/backfiller.rs | 176 +++++++++++- .../crates/event_indexer/src/error.rs | 3 + .../crates/event_indexer/src/indexer.rs | 10 +- .../crates/event_indexer/src/parser.rs | 46 +++- 8 files changed, 402 insertions(+), 112 deletions(-) diff --git a/observability/Cargo.lock b/observability/Cargo.lock index 8d997a5e..fa91a485 100644 --- a/observability/Cargo.lock +++ b/observability/Cargo.lock @@ -1608,6 +1608,7 @@ dependencies = [ "backoff", "bytemuck", "chrono", + "concurrent-queue", "crossbeam", "diesel", "dotenv", diff --git a/observability/crates/event_indexer/Cargo.toml b/observability/crates/event_indexer/Cargo.toml index d10334f4..3da0c318 100644 --- a/observability/crates/event_indexer/Cargo.toml +++ b/observability/crates/event_indexer/Cargo.toml @@ -24,6 +24,7 @@ anchor-lang = { workspace = true } backoff = { workspace = true } bytemuck = { workspace = true } chrono = { workspace = true } +concurrent-queue = "2.4.0" crossbeam = { workspace = true } diesel = { workspace = true, features = ["postgres", "chrono", "numeric"] } dotenv = { workspace = true } diff --git a/observability/crates/event_indexer/bin/backfill_events.rs b/observability/crates/event_indexer/bin/backfill_events.rs index 566c7ef4..026fe7c7 100644 --- a/observability/crates/event_indexer/bin/backfill_events.rs +++ b/observability/crates/event_indexer/bin/backfill_events.rs @@ -1,22 +1,38 @@ -use std::{panic, process, str::FromStr, thread::available_parallelism}; +use std::{ + panic, process, + str::FromStr, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread::available_parallelism, + time::Duration, +}; +use backoff::{exponential::ExponentialBackoffBuilder, retry, SystemClock}; use bytemuck::Contiguous; +use chrono::DateTime; +use concurrent_queue::ConcurrentQueue; use dotenv::dotenv; use envconfig::Envconfig; -use event_indexer::backfiller::{ - crawl_signatures_for_range, find_boundary_signatures_for_range, TransactionData, - MARGINFI_PROGRAM_GENESIS_SIG, -}; -use solana_client::{ - nonblocking::rpc_client::RpcClient, rpc_client::SerializableTransaction, - rpc_config::RpcBlockConfig, +use event_indexer::{ + backfiller::{ + crawl_signatures_for_range, generate_ranges, get_default_before_signature, Range, + TransactionData, MARGINFI_PROGRAM_GENESIS_SIG, + }, + db::establish_connection, + entity_store::EntityStore, + error::IndexingError, + parser::{MarginfiEvent, MarginfiEventParser, MarginfiEventWithMeta, MARGINFI_GROUP_ADDRESS}, }; +use rpc_utils::conversion::convert_encoded_ui_transaction; +use solana_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{ commitment_config::{CommitmentConfig, CommitmentLevel}, signature::Signature, }; -use solana_transaction_status::UiTransactionEncoding; -use tracing::info; +use tokio::time::interval; +use tracing::{error, info, warn}; use tracing_subscriber::{layer::SubscriberExt, EnvFilter}; #[derive(Envconfig, Debug, Clone)] @@ -29,6 +45,8 @@ pub struct Config { pub before: Option, #[envconfig(from = "UNTIL_SIGNATURE")] pub until: Option, + #[envconfig(from = "DATABASE_URL")] + pub database_url: String, #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] pub gcp_sa_key: String, #[envconfig(from = "PRETTY_LOGS")] @@ -68,44 +86,12 @@ pub async fn main() { }, ); - let latest_slot = rpc_client - .get_slot() - .await - .expect("Failed to fetch latest slot"); - let latest_block_slots = rpc_client - .get_blocks(latest_slot - 100, None) - .await - .expect("Failed to fetch block ids"); - if latest_block_slots.is_empty() { - panic!("Failed to find blocks in the last 100 slots"); - } - - let latest_block = rpc_client - .get_block_with_config( - *latest_block_slots.last().unwrap(), - RpcBlockConfig { - encoding: Some(UiTransactionEncoding::Base64), - max_supported_transaction_version: Some(0), - ..Default::default() - }, - ) - .await - .expect("Failed to fetch latest block"); - let before_sig = latest_block - .transactions - .unwrap() - .last() - .unwrap() - .transaction - .decode() - .unwrap() - .get_signature() - .clone(); - + let default_before_sig = get_default_before_signature(&rpc_client).await; + let before_sig = Signature::from_str(&config.before.unwrap_or(default_before_sig)).unwrap(); let until_sig = Signature::from_str( &config .until - .unwrap_or_else(|| MARGINFI_PROGRAM_GENESIS_SIG.to_string()), + .unwrap_or(MARGINFI_PROGRAM_GENESIS_SIG.to_string()), ) .unwrap(); @@ -113,19 +99,40 @@ pub async fn main() { .map(|c| c.into_integer() as usize) .unwrap_or(1); - let boundary_sigs = - find_boundary_signatures_for_range(&rpc_client, threads, before_sig, until_sig) + let mut tasks: Vec>> = vec![]; + + let range_queue = Arc::new(ConcurrentQueue::::bounded(threads * 2)); + let is_range_complete = Arc::new(AtomicBool::new(false)); + + let range_queue_clone = range_queue.clone(); + let rpc_endpoint_clone = rpc_endpoint.clone(); + let is_range_complete_clone = is_range_complete.clone(); + tasks.push(std::thread::spawn(move || { + tokio::runtime::Runtime::new().unwrap().block_on(async { + let rpc_client = RpcClient::new_with_commitment( + rpc_endpoint_clone, + CommitmentConfig { + commitment: CommitmentLevel::Confirmed, + }, + ); + generate_ranges( + rpc_client, + before_sig, + until_sig, + &range_queue_clone, + is_range_complete_clone, + ) .await - .unwrap(); + .map_err(|e| IndexingError::FailedToGenerateRange(e.to_string()))?; - info!("Boundary signatures: {:?}", boundary_sigs); + Ok(()) + }) + })); let (transaction_tx, transaction_rx) = crossbeam::channel::unbounded::(); - let mut tasks = vec![]; for i in 0..threads { - let until_sig = boundary_sigs[i]; - let before_sig = boundary_sigs[i + 1]; + info!("Spawning thread: {:?}", i); let local_transaction_tx = transaction_tx.clone(); let rpc_client = RpcClient::new_with_commitment( @@ -134,49 +141,129 @@ pub async fn main() { commitment: CommitmentLevel::Confirmed, }, ); - - info!( - "Spawning thread for range: {:?}..{:?}", - until_sig, before_sig - ); + let range_queue = range_queue.clone(); + let is_range_complete = is_range_complete.clone(); tasks.push(std::thread::spawn(move || { tokio::runtime::Runtime::new().unwrap().block_on(async { - crawl_signatures_for_range( - i as u64, - rpc_client, - marginfi::ID, - before_sig.1, - until_sig.1, - local_transaction_tx, - None, - ) - .await + let mut timer = interval(Duration::from_millis(200)); + while !is_range_complete.load(Ordering::Relaxed) { + if let Ok(Range { + before_sig, + until_sig, + before_slot, + until_slot, + progress, + }) = range_queue.pop() + { + info!( + "Thread {:?} got range: {:?} - {:?} ({:.2?}%)", + i, until_slot, before_slot, progress + ); + crawl_signatures_for_range( + i as u64, + &rpc_client, + marginfi::ID, + before_sig, + until_sig, + &local_transaction_tx, + None, + ) + .await?; + } + + timer.tick().await; + } + + Ok(()) }) })); } - let mut tx_count = 0; + let parser = MarginfiEventParser::new(marginfi::ID, MARGINFI_GROUP_ADDRESS); + let mut entity_store = EntityStore::new(rpc_endpoint, config.database_url.clone()); + let mut db_connection = establish_connection(config.database_url.clone()); + + let mut event_counter = 0; + let mut print_time = std::time::Instant::now(); + while let Ok(TransactionData { - task_id, transaction, + task_id, .. }) = transaction_rx.recv() { - tx_count += 1; - let until_slot = boundary_sigs[task_id as usize].0; - let before_slot = boundary_sigs[task_id as usize + 1].0; - println!( - "[{}] {:.1}% complete (total: {}, remaining ranges: {})", - task_id, - (before_slot - transaction.slot) as f64 / (before_slot - until_slot) as f64 * 100.0, - tx_count, - tasks.iter().fold(0, |mut acc, task| { - if !task.is_finished() { - acc = acc + 1; - } - acc - }) - ); + if tasks.iter().all(|task| task.is_finished()) { + break; + } + + let timestamp = transaction.block_time.unwrap(); + let slot = transaction.slot; + let versioned_tx_with_meta = + convert_encoded_ui_transaction(transaction.transaction).unwrap(); + + let events = parser.extract_events(timestamp, slot, versioned_tx_with_meta); + event_counter += events.len(); + + let elapsed = print_time.elapsed().as_secs(); + if elapsed > 30 { + warn!("Events processed: {:?}", event_counter); + print_time = std::time::Instant::now(); + } + + for MarginfiEventWithMeta { + event, + timestamp, + in_flashloan, + call_stack, + tx_sig, + } in events + { + let timestamp = DateTime::from_timestamp(timestamp, 0).unwrap().naive_utc(); + let tx_sig = tx_sig.to_string(); + let call_stack = serde_json::to_string( + &call_stack + .into_iter() + .map(|cs| cs.to_string()) + .collect::>(), + ) + .unwrap_or_else(|_| "null".to_string()); + + let mut retries = 0; + retry( + ExponentialBackoffBuilder::::new() + .with_max_interval(Duration::from_secs(5)) + .build(), + || match event.db_insert( + timestamp, + tx_sig.clone(), + in_flashloan, + call_stack.clone(), + &mut db_connection, + &mut entity_store, + ) { + Ok(signatures) => Ok(signatures), + Err(e) => { + if retries > 5 { + error!( + "[{:?}] Failed to insert event after 5 retries: {:?} - {:?} ({:?})", + task_id, event, e, tx_sig + ); + Err(backoff::Error::permanent(e)) + } else { + warn!( + "[{:?}] Failed to insert event, retrying: {:?} - {:?} ({:?})", + task_id, event, e, tx_sig + ); + retries += 1; + Err(backoff::Error::transient(e)) + } + } + }, + ) + .unwrap(); + } } + + info!("Done!"); } diff --git a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql index a47a02db..174d3b3e 100644 --- a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql +++ b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql @@ -5,29 +5,29 @@ CREATE TABLE "bank_operational_state" ( "name" VARCHAR NOT NULL ); -INSERT INTO "bank_operational_state" ("name") -VALUES ('Paused'), - ('Operational'), - ('ReduceOnly'); +INSERT INTO "bank_operational_state" ("id", "name") +VALUES (0, 'Paused'), + (1, 'Operational'), + (2, 'ReduceOnly'); CREATE TABLE "oracle_setup" ( "id" SERIAL PRIMARY KEY, "name" VARCHAR NOT NULL ); -INSERT INTO "oracle_setup" ("name") -VALUES ('None'), - ('PythEma'), - ('SwitchboardV2'); +INSERT INTO "oracle_setup" ("id", "name") +VALUES (0, 'None'), + (1, 'PythEma'), + (2, 'SwitchboardV2'); CREATE TABLE "risk_tier" ( "id" SERIAL PRIMARY KEY, "name" VARCHAR NOT NULL ); -INSERT INTO "risk_tier" ("name") -VALUES ('Collateral'), - ('Isolated'); +INSERT INTO "risk_tier" ("id", "name") +VALUES (0, 'Collateral'), + (1, 'Isolated'); -- Entities diff --git a/observability/crates/event_indexer/src/backfiller.rs b/observability/crates/event_indexer/src/backfiller.rs index 4620ff0a..7f641790 100644 --- a/observability/crates/event_indexer/src/backfiller.rs +++ b/observability/crates/event_indexer/src/backfiller.rs @@ -1,6 +1,7 @@ use backoff::{future::retry, ExponentialBackoffBuilder}; +use concurrent_queue::ConcurrentQueue; use crossbeam::channel::Sender; -use futures::{future::try_join_all, stream, StreamExt}; +use futures::{future::try_join_all, lock::Mutex, stream, StreamExt}; use solana_client::{ nonblocking::rpc_client::RpcClient, rpc_client::{GetConfirmedSignaturesForAddress2Config, SerializableTransaction}, @@ -10,13 +11,22 @@ use solana_client::{ use solana_rpc_client_api::client_error::{Error as ClientError, ErrorKind}; use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature}; use solana_transaction_status::{EncodedConfirmedTransactionWithStatusMeta, UiTransactionEncoding}; -use std::{str::FromStr, time::Duration}; +use std::{ + str::FromStr, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::time::interval; use tracing::{info, warn}; use crate::error::IndexingError; pub const MARGINFI_PROGRAM_GENESIS_SIG: &str = "36ViByXbDDgXHo3fTghrrT9zHMu9mYCJMhpKYTpytHcj7Nxkpb6gQdBJRxtDbF53mebNc8HR4aC7pcKmRNypxTWC"; +const SLOT_RANGE_SIZE: u64 = 5000; // ~40 minutes #[derive(Debug)] pub struct TransactionData { @@ -69,8 +79,8 @@ pub async fn find_boundary_signatures_for_range( let boundary_sigs = try_join_all(boundary_slots.into_iter().enumerate().map( |(index, slot)| async move { let forward = index != last_boundary_index; - let boundary_slot = find_boundary_signature(rpc_client, slot, forward, None).await?; - Ok((slot, boundary_slot)) + let boundary_sig = find_boundary_signature(rpc_client, slot, forward, None).await?; + Ok((slot, boundary_sig)) }, )) .await?; @@ -172,11 +182,11 @@ pub async fn find_boundary_signature( pub async fn crawl_signatures_for_range( task_id: u64, - rpc_client: RpcClient, + rpc_client: &RpcClient, address: Pubkey, before: Signature, until: Signature, - transaction_tx: Sender, + transaction_tx: &Sender, max_concurrent_requests: Option, ) -> Result<(), IndexingError> { let mut last_fetched_signature = before; @@ -277,3 +287,157 @@ pub async fn crawl_signatures_for_range( Ok(()) } + +#[derive(Debug)] +pub struct Range { + pub before_sig: Signature, + pub until_sig: Signature, + pub before_slot: u64, + pub until_slot: u64, + pub progress: f64, +} + +pub async fn generate_ranges( + rpc_client: RpcClient, + overall_before_sig: Signature, + overall_until_sig: Signature, + range_queue: &ConcurrentQueue, + is_range_complete: Arc, +) -> Result<(), Box> { + let sig_statuses = rpc_client + .get_signature_statuses_with_history(&[overall_before_sig, overall_until_sig]) + .await? + .value; + let (overall_before_slot, overall_until_slot) = ( + sig_statuses[0].as_ref().unwrap().slot, + sig_statuses[1].as_ref().unwrap().slot, + ); + + let mut cursor_slot = overall_until_slot; + + let mut timer = interval(Duration::from_millis(200)); + loop { + if range_queue.is_full() { + timer.tick().await; + continue; + } + + let remaining_capacity = range_queue.capacity().unwrap() - range_queue.len(); + println!("Remaining capacity: {}", remaining_capacity); + + let mut boundary_slots = vec![cursor_slot]; + for _ in 0..remaining_capacity { + cursor_slot = (cursor_slot + SLOT_RANGE_SIZE).min(overall_before_slot); + boundary_slots.push(cursor_slot); + + if cursor_slot == overall_before_slot { + break; + } + } + println!("Boundary slots: {:?}", boundary_slots); + + let rpc_client_ref = &rpc_client; + + let prev_cursor_sig = Arc::new(Mutex::new(overall_until_sig)); + let prev_cursor_slot = Arc::new(AtomicU64::new(boundary_slots.remove(0))); + + stream::iter( + boundary_slots + .iter() + .map(|slot| (*slot, prev_cursor_sig.clone(), prev_cursor_slot.clone())), + ) + .map(|(slot, prev_cursor_sig, prev_cursor_slot)| async move { + if slot == overall_before_slot { + (slot, overall_before_sig, prev_cursor_sig, prev_cursor_slot) + } else { + retry( + ExponentialBackoffBuilder::new() + .with_max_interval(Duration::from_secs(5)) + .build(), + || async { + match find_boundary_signature(rpc_client_ref, slot, true, None).await { + Ok(sig) => { + Ok((slot, sig, prev_cursor_sig.clone(), prev_cursor_slot.clone())) + } + Err(e) => Err(backoff::Error::transient(e)), + } + }, + ) + .await + .unwrap() + } + }) + .buffered(10) + .for_each( + |(cursor_slot, cursor_sig, prev_cursor_sig, prev_cursor_slot)| async move { + println!( + "Pushing range: {:?} [{:?} -> {:?}]", + prev_cursor_sig, prev_cursor_slot, cursor_slot, + ); + range_queue + .push(Range { + before_sig: cursor_sig, + before_slot: cursor_slot, + until_sig: prev_cursor_sig.lock().await.clone(), + until_slot: prev_cursor_slot.load(Ordering::Relaxed), + progress: ((cursor_slot as f64 - overall_until_slot as f64) + / (overall_before_slot as f64 - overall_until_slot as f64)) + * 100.0, + }) + .unwrap(); + + *prev_cursor_sig.lock().await = cursor_sig; // ok because we are doing an ordered buffering + prev_cursor_slot.store(cursor_slot, Ordering::Relaxed); + }, + ) + .await; + + if cursor_slot == overall_before_slot { + break; + } + + timer.tick().await; + } + + is_range_complete.store(true, Ordering::Relaxed); + + Ok(()) +} + +pub async fn get_default_before_signature(rpc_client: &RpcClient) -> String { + let latest_slot = rpc_client + .get_slot() + .await + .expect("Failed to fetch latest slot"); + let latest_block_slots = rpc_client + .get_blocks(latest_slot - 100, None) + .await + .expect("Failed to fetch block ids"); + if latest_block_slots.is_empty() { + panic!("Failed to find blocks in the last 100 slots"); + } + + let latest_block = rpc_client + .get_block_with_config( + *latest_block_slots.last().unwrap(), + RpcBlockConfig { + encoding: Some(UiTransactionEncoding::Base64), + max_supported_transaction_version: Some(0), + ..Default::default() + }, + ) + .await + .expect("Failed to fetch latest block"); + let before_sig = latest_block + .transactions + .unwrap() + .last() + .unwrap() + .transaction + .decode() + .unwrap() + .get_signature() + .clone(); + + before_sig.to_string() +} diff --git a/observability/crates/event_indexer/src/error.rs b/observability/crates/event_indexer/src/error.rs index 2d677c1e..0c2309bd 100644 --- a/observability/crates/event_indexer/src/error.rs +++ b/observability/crates/event_indexer/src/error.rs @@ -21,6 +21,9 @@ pub enum IndexingError { #[error("Failed to insert event: {0}")] FailedToInsertEvent(String), + #[error("Failed to generate range: {0}")] + FailedToGenerateRange(String), + #[error("An unknown error occurred")] Unknown, } diff --git a/observability/crates/event_indexer/src/indexer.rs b/observability/crates/event_indexer/src/indexer.rs index cf2421b2..81073149 100644 --- a/observability/crates/event_indexer/src/indexer.rs +++ b/observability/crates/event_indexer/src/indexer.rs @@ -297,7 +297,6 @@ async fn store_events( .collect::>(), ) .unwrap_or_else(|_| "null".to_string()); - println!("Storing event: {:?}", event); let mut retries = 0; retry( @@ -316,12 +315,15 @@ async fn store_events( Err(e) => { if retries > 5 { error!( - "Failed to insert event after 5 retries: {:?} - {:?}", - event, e + "Failed to insert event after 5 retries: {:?} - {:?} ({:?})", + event, e, tx_sig ); Err(backoff::Error::permanent(e)) } else { - warn!("Failed to insert event, retrying: {:?}", e); + warn!( + "Failed to insert event, retrying: {:?} ({:?})", + e, tx_sig + ); retries += 1; Err(backoff::Error::transient(e)) } diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 997bd593..07847ec4 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, vec}; use anchor_lang::{AnchorDeserialize, Discriminator}; use chrono::NaiveDateTime; @@ -24,10 +24,9 @@ use solana_sdk::{ hash::Hash, instruction::CompiledInstruction, message::SimpleAddressLoader, - pubkey, - pubkey::Pubkey, signature::Signature, transaction::{MessageHash, SanitizedTransaction}, + {pubkey, pubkey::Pubkey}, }; use solana_transaction_status::{ InnerInstruction, InnerInstructions, VersionedTransactionWithStatusMeta, @@ -44,6 +43,10 @@ use crate::{ const SPL_TRANSFER_DISCRIMINATOR: u8 = 3; pub const MARGINFI_GROUP_ADDRESS: Pubkey = pubkey!("4qp6Fx6tnZkY5Wropq9wUYgtFxXKwE6viZxFHg3rdAG8"); const COMPACT_BANK_CONFIG_ARG_UPGRADE_SLOT: u64 = 232_933_019; +const TOTAL_ASSET_VALUE_INIT_LIMIT_UPGRADE_SLOT: u64 = 204_502_867; +const ADD_BANK_IX_ACCOUNTS_CHANGE_UPGRADE_SLOT: u64 = 232_933_019; +const RISK_TIER_UPGRADE_SLOT: u64 = 179_862_046; +const INTEREST_RATE_CONFIG_UPGRADE_SLOT: u64 = 178_870_399; #[derive(Debug)] pub struct MarginfiEventWithMeta { @@ -1763,8 +1766,11 @@ impl MarginfiEventParser { .into() }; - let bank_mint = *ix_accounts.get(3).unwrap(); - let bank = *ix_accounts.get(4).unwrap(); + let (bank_mint, bank) = if slot < ADD_BANK_IX_ACCOUNTS_CHANGE_UPGRADE_SLOT { + (*ix_accounts.get(2).unwrap(), *ix_accounts.get(3).unwrap()) + } else { + (*ix_accounts.get(3).unwrap(), *ix_accounts.get(4).unwrap()) + }; Some(Event::AddBank(AddBankEvent { bank, @@ -1797,8 +1803,34 @@ impl MarginfiEventParser { return None; } - let bank_config_opt = - BankConfigOpt::deserialize(&mut &instruction_data[..363]).unwrap(); + // println!("Instruction data: {:?}", instruction_data); + // println!("data len: {:?}", instruction_data.len()); + + // let parsed = BankConfigOpt { + // interest_rate_config: Some(InterestRateConfigOpt { + // optimal_utilization_rate: Some(I80F48::from_num(0.95).into()), + // plateau_interest_rate: Some(I80F48::from_num(0.05).into()), + // max_interest_rate: Some(I80F48::from_num(2).into()), + // insurance_fee_fixed_apr: Some(I80F48::from_num(0).into()), + // ..Default::default() + // }), + // ..Default::default() + // }; + // println!("Parsed: {:?}", parsed); + // let ser = parsed.try_to_vec().unwrap(); + // println!("Serialized: {:?}", ser); + + let mut data = vec![]; + data.extend_from_slice(&instruction_data); + if slot < INTEREST_RATE_CONFIG_UPGRADE_SLOT { + data.extend_from_slice(&[0, 0, 0, 0]); + } else if slot < RISK_TIER_UPGRADE_SLOT { + data.extend_from_slice(&[0, 0, 0]); + } else if slot < TOTAL_ASSET_VALUE_INIT_LIMIT_UPGRADE_SLOT { + data.extend_from_slice(&[0]); + } + + let bank_config_opt = BankConfigOpt::deserialize(&mut data.as_slice()).unwrap(); let bank = *ix_accounts.get(2).unwrap(); From 3b3ab35c2469b7137f568ac0097bf40d6f7a6a95 Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 4 Apr 2024 10:44:23 +0800 Subject: [PATCH 12/20] fix: missing ranges at the end --- .../event_indexer/bin/backfill_events.rs | 49 +++++++++++++------ .../crates/event_indexer/src/backfiller.rs | 24 ++++----- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/observability/crates/event_indexer/bin/backfill_events.rs b/observability/crates/event_indexer/bin/backfill_events.rs index 026fe7c7..8e80f916 100644 --- a/observability/crates/event_indexer/bin/backfill_events.rs +++ b/observability/crates/event_indexer/bin/backfill_events.rs @@ -13,6 +13,7 @@ use backoff::{exponential::ExponentialBackoffBuilder, retry, SystemClock}; use bytemuck::Contiguous; use chrono::DateTime; use concurrent_queue::ConcurrentQueue; +use crossbeam::channel::TryRecvError; use dotenv::dotenv; use envconfig::Envconfig; use event_indexer::{ @@ -125,6 +126,8 @@ pub async fn main() { .await .map_err(|e| IndexingError::FailedToGenerateRange(e.to_string()))?; + info!("Finished generating ranges"); + Ok(()) }) })); @@ -147,7 +150,7 @@ pub async fn main() { tasks.push(std::thread::spawn(move || { tokio::runtime::Runtime::new().unwrap().block_on(async { let mut timer = interval(Duration::from_millis(200)); - while !is_range_complete.load(Ordering::Relaxed) { + while range_queue.len() > 0 || !is_range_complete.load(Ordering::Relaxed) { if let Ok(Range { before_sig, until_sig, @@ -157,8 +160,8 @@ pub async fn main() { }) = range_queue.pop() { info!( - "Thread {:?} got range: {:?} - {:?} ({:.2?}%)", - i, until_slot, before_slot, progress + "[{:?}] Processing {:?} -> {:?} ({:?} -> {:?})", + i, until_slot, before_slot, until_sig, before_sig ); crawl_signatures_for_range( i as u64, @@ -170,11 +173,17 @@ pub async fn main() { None, ) .await?; + info!( + "[{:?}] {:.2?}% completed {:?} - {:?} ({:?} -> {:?})", + i, progress, until_slot, before_slot, until_sig, before_sig + ); } timer.tick().await; } + info!("Thread {:?} done", i); + Ok(()) }) })); @@ -187,15 +196,29 @@ pub async fn main() { let mut event_counter = 0; let mut print_time = std::time::Instant::now(); - while let Ok(TransactionData { - transaction, - task_id, - .. - }) = transaction_rx.recv() - { - if tasks.iter().all(|task| task.is_finished()) { - break; - } + loop { + let maybe_item = transaction_rx.try_recv(); + + let TransactionData { + transaction, + task_id, + .. + } = match maybe_item { + Err(TryRecvError::Empty) => { + if tasks.iter().all(|task| task.is_finished()) { + info!("Done!"); + break; + } else { + std::thread::sleep(Duration::from_millis(100)); + continue; + } + } + Err(TryRecvError::Disconnected) => { + error!("Transaction channel disconnected! Exiting..."); + break; + } + Ok(tx_data) => tx_data, + }; let timestamp = transaction.block_time.unwrap(); let slot = transaction.slot; @@ -264,6 +287,4 @@ pub async fn main() { .unwrap(); } } - - info!("Done!"); } diff --git a/observability/crates/event_indexer/src/backfiller.rs b/observability/crates/event_indexer/src/backfiller.rs index 7f641790..476a8ca2 100644 --- a/observability/crates/event_indexer/src/backfiller.rs +++ b/observability/crates/event_indexer/src/backfiller.rs @@ -305,15 +305,21 @@ pub async fn generate_ranges( is_range_complete: Arc, ) -> Result<(), Box> { let sig_statuses = rpc_client - .get_signature_statuses_with_history(&[overall_before_sig, overall_until_sig]) + .get_signature_statuses_with_history(&[overall_until_sig, overall_before_sig]) .await? .value; - let (overall_before_slot, overall_until_slot) = ( + let (overall_until_slot, overall_before_slot) = ( sig_statuses[0].as_ref().unwrap().slot, sig_statuses[1].as_ref().unwrap().slot, ); + info!( + "Starting to generate ranges between {:?} ({:?}) and {:?} ({:?})", + overall_until_sig, overall_until_slot, overall_before_sig, overall_before_slot + ); + let mut cursor_slot = overall_until_slot; + let prev_cursor_sig = Arc::new(Mutex::new(overall_until_sig)); let mut timer = interval(Duration::from_millis(200)); loop { @@ -323,7 +329,6 @@ pub async fn generate_ranges( } let remaining_capacity = range_queue.capacity().unwrap() - range_queue.len(); - println!("Remaining capacity: {}", remaining_capacity); let mut boundary_slots = vec![cursor_slot]; for _ in 0..remaining_capacity { @@ -334,11 +339,8 @@ pub async fn generate_ranges( break; } } - println!("Boundary slots: {:?}", boundary_slots); let rpc_client_ref = &rpc_client; - - let prev_cursor_sig = Arc::new(Mutex::new(overall_until_sig)); let prev_cursor_slot = Arc::new(AtomicU64::new(boundary_slots.remove(0))); stream::iter( @@ -370,15 +372,13 @@ pub async fn generate_ranges( .buffered(10) .for_each( |(cursor_slot, cursor_sig, prev_cursor_sig, prev_cursor_slot)| async move { - println!( - "Pushing range: {:?} [{:?} -> {:?}]", - prev_cursor_sig, prev_cursor_slot, cursor_slot, - ); + let mut until_sig = prev_cursor_sig.lock().await; + range_queue .push(Range { before_sig: cursor_sig, before_slot: cursor_slot, - until_sig: prev_cursor_sig.lock().await.clone(), + until_sig: until_sig.clone(), until_slot: prev_cursor_slot.load(Ordering::Relaxed), progress: ((cursor_slot as f64 - overall_until_slot as f64) / (overall_before_slot as f64 - overall_until_slot as f64)) @@ -386,7 +386,7 @@ pub async fn generate_ranges( }) .unwrap(); - *prev_cursor_sig.lock().await = cursor_sig; // ok because we are doing an ordered buffering + *until_sig = cursor_sig; // ok because we are doing an ordered buffering prev_cursor_slot.store(cursor_slot, Ordering::Relaxed); }, ) From 8eaf6cb89b8a0c59d8b61fa2b8bdb4138f1341fc Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 4 Apr 2024 13:12:23 +0800 Subject: [PATCH 13/20] feat: index unknown events --- .../event_indexer/bin/backfill_events.rs | 2 +- .../2024-03-14-151141_initial_tables/up.sql | 12 +++++ .../crates/event_indexer/src/db/models.rs | 11 +++++ .../crates/event_indexer/src/db/schema.rs | 13 +++++ .../crates/event_indexer/src/parser.rs | 49 ++++++++++++++++--- 5 files changed, 78 insertions(+), 9 deletions(-) diff --git a/observability/crates/event_indexer/bin/backfill_events.rs b/observability/crates/event_indexer/bin/backfill_events.rs index 8e80f916..3234537e 100644 --- a/observability/crates/event_indexer/bin/backfill_events.rs +++ b/observability/crates/event_indexer/bin/backfill_events.rs @@ -206,7 +206,7 @@ pub async fn main() { } = match maybe_item { Err(TryRecvError::Empty) => { if tasks.iter().all(|task| task.is_finished()) { - info!("Done!"); + info!("Done! {:?} events processed", event_counter); break; } else { std::thread::sleep(Duration::from_millis(100)); diff --git a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql index 174d3b3e..46fdf05d 100644 --- a/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql +++ b/observability/crates/event_indexer/migrations/2024-03-14-151141_initial_tables/up.sql @@ -73,6 +73,18 @@ SELECT diesel_manage_updated_at('accounts'); -- Events +CREATE TABLE "unknown_events"( + "id" SERIAL PRIMARY KEY, + "timestamp" TIMESTAMP NOT NULL, + "tx_sig" VARCHAR NOT NULL, + "in_flashloan" BOOLEAN NOT NULL, + "call_stack" VARCHAR NOT NULL, + + "created_at" TIMESTAMP NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP NOT NULL DEFAULT NOW() +); +SELECT diesel_manage_updated_at('unknown_events'); + CREATE TABLE "create_account_events"( "id" SERIAL PRIMARY KEY, "timestamp" TIMESTAMP NOT NULL, diff --git a/observability/crates/event_indexer/src/db/models.rs b/observability/crates/event_indexer/src/db/models.rs index 29877ec8..49cd6b86 100644 --- a/observability/crates/event_indexer/src/db/models.rs +++ b/observability/crates/event_indexer/src/db/models.rs @@ -40,6 +40,17 @@ pub struct Accounts { pub user_id: i32, } +#[derive(Default, Debug, Queryable, Selectable, Insertable)] +#[diesel(table_name = unknown_events)] +pub struct UnknownEvents { + #[diesel(skip_insertion)] + pub id: i32, + pub timestamp: chrono::NaiveDateTime, + pub tx_sig: String, + pub in_flashloan: bool, + pub call_stack: String, +} + #[derive(Default, Debug, Queryable, Selectable, Insertable, Associations)] #[diesel(table_name = create_account_events)] #[diesel(belongs_to(Accounts, foreign_key = account_id), belongs_to(Users, foreign_key = authority_id))] diff --git a/observability/crates/event_indexer/src/db/schema.rs b/observability/crates/event_indexer/src/db/schema.rs index de5d41d4..873c4ae8 100644 --- a/observability/crates/event_indexer/src/db/schema.rs +++ b/observability/crates/event_indexer/src/db/schema.rs @@ -215,6 +215,18 @@ diesel::table! { } } +diesel::table! { + unknown_events (id) { + id -> Int4, + timestamp -> Timestamp, + tx_sig -> Varchar, + in_flashloan -> Bool, + call_stack -> Varchar, + created_at -> Timestamp, + updated_at -> Timestamp, + } +} + diesel::table! { users (id) { id -> Int4, @@ -306,6 +318,7 @@ diesel::allow_tables_to_appear_in_same_query!( repay_events, risk_tier, transfer_account_authority_events, + unknown_events, users, withdraw_emissions_events, withdraw_events, diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 07847ec4..0c7a2a5f 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -10,10 +10,9 @@ use enum_dispatch::enum_dispatch; use fixed::types::I80F48; use marginfi::{ instruction::{ - LendingAccountBorrow, LendingAccountCloseBalance, LendingAccountDeposit, - LendingAccountEndFlashloan, LendingAccountLiquidate, LendingAccountRepay, - LendingAccountSettleEmissions, LendingAccountStartFlashloan, LendingAccountWithdraw, - LendingAccountWithdrawEmissions, LendingPoolAccrueBankInterest, LendingPoolAddBank, + LendingAccountBorrow, LendingAccountDeposit, LendingAccountEndFlashloan, + LendingAccountLiquidate, LendingAccountRepay, LendingAccountStartFlashloan, + LendingAccountWithdraw, LendingAccountWithdrawEmissions, LendingPoolAddBank, LendingPoolAddBankWithSeed, LendingPoolConfigureBank, MarginfiAccountInitialize, SetNewAccountAuthority, }, @@ -86,6 +85,43 @@ pub enum Event { // Admin actions AddBank(AddBankEvent), ConfigureBank(ConfigureBankEvent), + + Unknown(UnknownEvent), +} + +#[derive(Debug)] +pub struct UnknownEvent {} + +impl MarginfiEvent for UnknownEvent { + fn db_insert( + &self, + timestamp: NaiveDateTime, + tx_sig: String, + in_flashloan: bool, + call_stack: String, + db_connection: &mut PgConnection, + _entity_store: &mut EntityStore, + ) -> Result<(), IndexingError> { + let id: Option = diesel::insert_into(unknown_events::table) + .values(&UnknownEvents { + timestamp, + tx_sig, + call_stack, + in_flashloan, + ..Default::default() + }) + .on_conflict_do_nothing() + .returning(unknown_events::id) + .get_result(db_connection) + .optional() + .map_err(|err| IndexingError::FailedToInsertEvent(err.to_string()))?; + + if id.is_none() { + info!("event already exists"); + } + + Ok(()) + } } #[derive(Debug)] @@ -1849,15 +1885,12 @@ impl MarginfiEventParser { None } - LendingAccountCloseBalance::DISCRIMINATOR - | LendingPoolAccrueBankInterest::DISCRIMINATOR - | LendingAccountSettleEmissions::DISCRIMINATOR => None, _ => { warn!( "Unknown instruction discriminator {:?} in {:?}", discriminator, tx_signature ); - None + Some(Event::Unknown(UnknownEvent {})) } } } From b1c4c8420d9a30d469611409f5ba305b3ca30022 Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:14:27 +0800 Subject: [PATCH 14/20] fix: ensure boundary txs are not mfi txs --- .../event_indexer/bin/backfill_events.rs | 1 + .../crates/event_indexer/src/backfiller.rs | 63 ++++++++++++++++--- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/observability/crates/event_indexer/bin/backfill_events.rs b/observability/crates/event_indexer/bin/backfill_events.rs index 3234537e..9febb0d6 100644 --- a/observability/crates/event_indexer/bin/backfill_events.rs +++ b/observability/crates/event_indexer/bin/backfill_events.rs @@ -117,6 +117,7 @@ pub async fn main() { }, ); generate_ranges( + &marginfi::ID, rpc_client, before_sig, until_sig, diff --git a/observability/crates/event_indexer/src/backfiller.rs b/observability/crates/event_indexer/src/backfiller.rs index 476a8ca2..8257845f 100644 --- a/observability/crates/event_indexer/src/backfiller.rs +++ b/observability/crates/event_indexer/src/backfiller.rs @@ -2,6 +2,7 @@ use backoff::{future::retry, ExponentialBackoffBuilder}; use concurrent_queue::ConcurrentQueue; use crossbeam::channel::Sender; use futures::{future::try_join_all, lock::Mutex, stream, StreamExt}; +use rpc_utils::conversion::convert_encoded_ui_transaction; use solana_client::{ nonblocking::rpc_client::RpcClient, rpc_client::{GetConfirmedSignaturesForAddress2Config, SerializableTransaction}, @@ -9,7 +10,14 @@ use solana_client::{ rpc_request::RpcError, }; use solana_rpc_client_api::client_error::{Error as ClientError, ErrorKind}; -use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature}; +use solana_sdk::{ + commitment_config::CommitmentConfig, + hash::Hash, + message::SimpleAddressLoader, + pubkey::Pubkey, + signature::Signature, + transaction::{MessageHash, SanitizedTransaction}, +}; use solana_transaction_status::{EncodedConfirmedTransactionWithStatusMeta, UiTransactionEncoding}; use std::{ str::FromStr, @@ -36,6 +44,7 @@ pub struct TransactionData { } pub async fn find_boundary_signatures_for_range( + program_id: &Pubkey, rpc_client: &RpcClient, task_count: usize, before: Signature, @@ -79,7 +88,8 @@ pub async fn find_boundary_signatures_for_range( let boundary_sigs = try_join_all(boundary_slots.into_iter().enumerate().map( |(index, slot)| async move { let forward = index != last_boundary_index; - let boundary_sig = find_boundary_signature(rpc_client, slot, forward, None).await?; + let boundary_sig = + find_boundary_signature(program_id, rpc_client, slot, forward, None).await?; Ok((slot, boundary_sig)) }, )) @@ -89,6 +99,7 @@ pub async fn find_boundary_signatures_for_range( } pub async fn find_boundary_signature( + program_id: &Pubkey, rpc_client: &RpcClient, slot: u64, forward: bool, @@ -113,12 +124,45 @@ pub async fn find_boundary_signature( match result { Ok(block) => { - if let Some(boundary_txs) = block.transactions { - let boundary_tx = if forward { - boundary_txs.last().unwrap() - } else { - boundary_txs.first().unwrap() + if let Some(mut boundary_txs) = block.transactions { + let mut counter = 0; + // Look for a non-mfi transaction in the block, as the boundary + let boundary_tx = loop { + if forward { + boundary_txs.reverse(); + } + let mut boundary_txs_iter = boundary_txs.iter(); + + let Some(boundary_tx) = boundary_txs_iter.next() else { + panic!("ONLY MFI TXS IN BLOCK FOR SLOT {}???!!!", candidate_slot); + }; + + let versioned_tx_with_meta = + convert_encoded_ui_transaction(boundary_tx.clone()).unwrap(); + let sanitized_tx = SanitizedTransaction::try_create( + versioned_tx_with_meta.transaction, + MessageHash::Precomputed(Hash::default()), + None, + SimpleAddressLoader::Enabled( + versioned_tx_with_meta.meta.loaded_addresses, + ), + true, + ) + .unwrap(); + + println!("pass {}", counter); + counter += 1; + + if !sanitized_tx + .message() + .account_keys() + .iter() + .any(|key| key == program_id) + { + break boundary_tx; + } }; + let boundary_sig = boundary_tx .transaction .decode() @@ -298,6 +342,7 @@ pub struct Range { } pub async fn generate_ranges( + program_id: &Pubkey, rpc_client: RpcClient, overall_before_sig: Signature, overall_until_sig: Signature, @@ -357,7 +402,9 @@ pub async fn generate_ranges( .with_max_interval(Duration::from_secs(5)) .build(), || async { - match find_boundary_signature(rpc_client_ref, slot, true, None).await { + match find_boundary_signature(program_id, rpc_client_ref, slot, true, None) + .await + { Ok(sig) => { Ok((slot, sig, prev_cursor_sig.clone(), prev_cursor_slot.clone())) } From 6f49651e32a45a35350586dd81ee25aa08310bb6 Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:15:02 +0800 Subject: [PATCH 15/20] fix: remove log --- observability/crates/event_indexer/src/backfiller.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/observability/crates/event_indexer/src/backfiller.rs b/observability/crates/event_indexer/src/backfiller.rs index 8257845f..2f821de5 100644 --- a/observability/crates/event_indexer/src/backfiller.rs +++ b/observability/crates/event_indexer/src/backfiller.rs @@ -125,7 +125,6 @@ pub async fn find_boundary_signature( match result { Ok(block) => { if let Some(mut boundary_txs) = block.transactions { - let mut counter = 0; // Look for a non-mfi transaction in the block, as the boundary let boundary_tx = loop { if forward { @@ -150,9 +149,6 @@ pub async fn find_boundary_signature( ) .unwrap(); - println!("pass {}", counter); - counter += 1; - if !sanitized_tx .message() .account_keys() From 1af7dea11b5917ce9360ca0afe2cda2ec653e959 Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:36:06 +0800 Subject: [PATCH 16/20] fix: handle empty blocks --- .../crates/event_indexer/src/backfiller.rs | 116 +++++++++--------- 1 file changed, 61 insertions(+), 55 deletions(-) diff --git a/observability/crates/event_indexer/src/backfiller.rs b/observability/crates/event_indexer/src/backfiller.rs index 2f821de5..89361c4f 100644 --- a/observability/crates/event_indexer/src/backfiller.rs +++ b/observability/crates/event_indexer/src/backfiller.rs @@ -110,7 +110,20 @@ pub async fn find_boundary_signature( let mut i = 0; let mut candidate_slot = slot; - loop { + 'block_search: loop { + if i > 0 { + i += 1; + if forward { + candidate_slot += i; + } else { + candidate_slot -= i; + } + + if i > max_retries { + return Err(IndexingError::BoundarySignatureNotFound(slot, max_retries)); + } + } + let result = rpc_client .get_block_with_config( candidate_slot, @@ -124,52 +137,57 @@ pub async fn find_boundary_signature( match result { Ok(block) => { - if let Some(mut boundary_txs) = block.transactions { - // Look for a non-mfi transaction in the block, as the boundary - let boundary_tx = loop { - if forward { - boundary_txs.reverse(); - } - let mut boundary_txs_iter = boundary_txs.iter(); - - let Some(boundary_tx) = boundary_txs_iter.next() else { - panic!("ONLY MFI TXS IN BLOCK FOR SLOT {}???!!!", candidate_slot); - }; - - let versioned_tx_with_meta = - convert_encoded_ui_transaction(boundary_tx.clone()).unwrap(); - let sanitized_tx = SanitizedTransaction::try_create( - versioned_tx_with_meta.transaction, - MessageHash::Precomputed(Hash::default()), - None, - SimpleAddressLoader::Enabled( - versioned_tx_with_meta.meta.loaded_addresses, - ), - true, - ) - .unwrap(); + let Some(mut boundary_txs) = block.transactions else { + warn!("No transactions in block for slot {}", candidate_slot); + continue; + }; - if !sanitized_tx - .message() - .account_keys() - .iter() - .any(|key| key == program_id) - { - break boundary_tx; - } + if boundary_txs.is_empty() { + warn!("No transactions in block for slot {}", candidate_slot); + continue; + } + + // Look for a non-mfi transaction in the block, as the boundary + let boundary_tx = loop { + if forward { + boundary_txs.reverse(); + } + let mut boundary_txs_iter = boundary_txs.iter(); + + let Some(boundary_tx) = boundary_txs_iter.next() else { + warn!("ONLY MFI TXS IN BLOCK FOR SLOT {}???!!!", candidate_slot); + continue 'block_search; }; - let boundary_sig = boundary_tx - .transaction - .decode() - .unwrap() - .get_signature() - .clone(); + let versioned_tx_with_meta = + convert_encoded_ui_transaction(boundary_tx.clone()).unwrap(); + let sanitized_tx = SanitizedTransaction::try_create( + versioned_tx_with_meta.transaction, + MessageHash::Precomputed(Hash::default()), + None, + SimpleAddressLoader::Enabled(versioned_tx_with_meta.meta.loaded_addresses), + true, + ) + .unwrap(); - return Ok(boundary_sig); - } else { - warn!("No transactions in block for slot {}", candidate_slot); - } + if !sanitized_tx + .message() + .account_keys() + .iter() + .any(|key| key == program_id) + { + break boundary_tx; + } + }; + + let boundary_sig = boundary_tx + .transaction + .decode() + .unwrap() + .get_signature() + .clone(); + + return Ok(boundary_sig); } Err(error) => { match error { @@ -205,18 +223,6 @@ pub async fn find_boundary_signature( }; } } - - i += 1; - if forward { - candidate_slot += i; - } else { - candidate_slot -= i; - } - - if i > max_retries { - return Err(IndexingError::BoundarySignatureNotFound(slot, max_retries)); - } - continue; } } From 6b8863d54a337349568a8a2ae87574e560e95cc6 Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:39:31 +0800 Subject: [PATCH 17/20] fix: handle empty blocks --- observability/crates/event_indexer/src/backfiller.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/observability/crates/event_indexer/src/backfiller.rs b/observability/crates/event_indexer/src/backfiller.rs index 89361c4f..ad9c8d10 100644 --- a/observability/crates/event_indexer/src/backfiller.rs +++ b/observability/crates/event_indexer/src/backfiller.rs @@ -109,9 +109,10 @@ pub async fn find_boundary_signature( let mut i = 0; let mut candidate_slot = slot; + let mut first = true; 'block_search: loop { - if i > 0 { + if !first { i += 1; if forward { candidate_slot += i; @@ -123,6 +124,7 @@ pub async fn find_boundary_signature( return Err(IndexingError::BoundarySignatureNotFound(slot, max_retries)); } } + first = false; let result = rpc_client .get_block_with_config( From eccc21d1fc6282506896718c2f195ef80968869f Mon Sep 17 00:00:00 2001 From: man0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 4 Apr 2024 21:52:37 +0800 Subject: [PATCH 18/20] feat: chillout on logs --- observability/crates/event_indexer/src/parser.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/observability/crates/event_indexer/src/parser.rs b/observability/crates/event_indexer/src/parser.rs index 0c7a2a5f..f0b603b5 100644 --- a/observability/crates/event_indexer/src/parser.rs +++ b/observability/crates/event_indexer/src/parser.rs @@ -1885,13 +1885,7 @@ impl MarginfiEventParser { None } - _ => { - warn!( - "Unknown instruction discriminator {:?} in {:?}", - discriminator, tx_signature - ); - Some(Event::Unknown(UnknownEvent {})) - } + _ => Some(Event::Unknown(UnknownEvent {})), } } } From 37684ab60b7248ec0e804055fc3ff0398a74f22b Mon Sep 17 00:00:00 2001 From: losman0s <95379755+losman0s@users.noreply.github.com> Date: Thu, 4 Apr 2024 23:02:20 +0800 Subject: [PATCH 19/20] bootstrap price indexer --- observability/Cargo.lock | 14 +++++ .../crates/solana-price-indexer/Cargo.toml | 18 ++++++ .../bin/index_solana_prices.rs | 58 +++++++++++++++++++ .../crates/solana-price-indexer/src/error.rs | 7 +++ .../solana-price-indexer/src/indexer.rs | 13 +++++ .../crates/solana-price-indexer/src/lib.rs | 2 + 6 files changed, 112 insertions(+) create mode 100644 observability/crates/solana-price-indexer/Cargo.toml create mode 100644 observability/crates/solana-price-indexer/bin/index_solana_prices.rs create mode 100644 observability/crates/solana-price-indexer/src/error.rs create mode 100644 observability/crates/solana-price-indexer/src/indexer.rs create mode 100644 observability/crates/solana-price-indexer/src/lib.rs diff --git a/observability/Cargo.lock b/observability/Cargo.lock index fa91a485..6f9693e7 100644 --- a/observability/Cargo.lock +++ b/observability/Cargo.lock @@ -4660,6 +4660,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "solana_price_indexer" +version = "0.1.0" +dependencies = [ + "backoff", + "dotenv", + "envconfig", + "thiserror", + "tokio", + "tracing", + "tracing-stackdriver", + "tracing-subscriber", +] + [[package]] name = "solana_rbpf" version = "0.6.1" diff --git a/observability/crates/solana-price-indexer/Cargo.toml b/observability/crates/solana-price-indexer/Cargo.toml new file mode 100644 index 00000000..26dd7abb --- /dev/null +++ b/observability/crates/solana-price-indexer/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "solana_price_indexer" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "index-solana-prices" +path = "bin/index_solana_prices.rs" + +[dependencies] +backoff = { workspace = true } +dotenv = { workspace = true } +envconfig = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-stackdriver = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/observability/crates/solana-price-indexer/bin/index_solana_prices.rs b/observability/crates/solana-price-indexer/bin/index_solana_prices.rs new file mode 100644 index 00000000..ff16d54c --- /dev/null +++ b/observability/crates/solana-price-indexer/bin/index_solana_prices.rs @@ -0,0 +1,58 @@ +use dotenv::dotenv; +use envconfig::Envconfig; +use solana_price_indexer::{error::IndexingError, indexer::SolanaPriceIndexer}; +use tracing_subscriber::{layer::SubscriberExt, EnvFilter}; + +use std::{panic, process}; + +#[derive(Envconfig, Debug, Clone)] +pub struct Config { + #[envconfig(from = "RPC_HOST")] + pub rpc_host: String, + #[envconfig(from = "RPC_TOKEN")] + pub rpc_token: String, + #[envconfig(from = "MONITOR_INTERVAL")] + pub monitor_interval: u64, + #[envconfig(from = "PRETTY_LOGS")] + pub pretty_logs: Option, + #[envconfig(from = "DATABASE_URL")] + pub database_url: String, + // #[envconfig(from = "INDEX_TRANSACTIONS_PROJECT_ID")] + // pub project_id: String, + // #[envconfig(from = "INDEX_TRANSACTIONS_PUBSUB_TOPIC_NAME")] + // pub topic_name: String, + // #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] + // pub gcp_sa_key: Option, +} + +#[tokio::main] +pub async fn main() -> Result<(), IndexingError> { + dotenv().ok(); + + let orig_hook = panic::take_hook(); + panic::set_hook(Box::new(move |panic_info| { + orig_hook(panic_info); + process::exit(1); + })); + + let config = Config::init_from_env().unwrap(); + + let pretty_logs = config.pretty_logs.unwrap_or(false); + + let filter = EnvFilter::from_default_env(); + let stackdriver = tracing_stackdriver::layer(); // writes to std::io::Stdout + let subscriber = tracing_subscriber::registry().with(filter); + if pretty_logs { + let subscriber = subscriber.with(tracing_subscriber::fmt::layer().compact()); + tracing::subscriber::set_global_default(subscriber).unwrap(); + } else { + let subscriber = subscriber.with(stackdriver); + tracing::subscriber::set_global_default(subscriber).unwrap(); + }; + + let mut indexer = SolanaPriceIndexer::new(); + + indexer.run().await; + + Ok(()) +} diff --git a/observability/crates/solana-price-indexer/src/error.rs b/observability/crates/solana-price-indexer/src/error.rs new file mode 100644 index 00000000..5535aefb --- /dev/null +++ b/observability/crates/solana-price-indexer/src/error.rs @@ -0,0 +1,7 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum IndexingError { + #[error("An unknown error occurred")] + Unknown, +} diff --git a/observability/crates/solana-price-indexer/src/indexer.rs b/observability/crates/solana-price-indexer/src/indexer.rs new file mode 100644 index 00000000..389ad936 --- /dev/null +++ b/observability/crates/solana-price-indexer/src/indexer.rs @@ -0,0 +1,13 @@ +pub struct SolanaPriceIndexer {} + +impl SolanaPriceIndexer { + pub fn new() -> Self { + SolanaPriceIndexer {} + } + + pub async fn run(&mut self) { + loop { + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } + } +} diff --git a/observability/crates/solana-price-indexer/src/lib.rs b/observability/crates/solana-price-indexer/src/lib.rs new file mode 100644 index 00000000..91f2f48b --- /dev/null +++ b/observability/crates/solana-price-indexer/src/lib.rs @@ -0,0 +1,2 @@ +pub mod error; +pub mod indexer; From b73d4b4d3dd9d06f4b35f55412999d21e9d98e2c Mon Sep 17 00:00:00 2001 From: losman0s <95379755+losman0s@users.noreply.github.com> Date: Fri, 5 Apr 2024 13:43:51 +0800 Subject: [PATCH 20/20] feat: wip price indexer --- observability/Cargo.lock | 303 ++++++++++++++++-- .../crates/solana-price-indexer/Cargo.toml | 5 + .../bin/index_solana_prices.rs | 4 +- .../solana-price-indexer/src/indexer.rs | 196 ++++++++++- .../solana-price-indexer/src/jupiter.rs | 141 ++++++++ .../crates/solana-price-indexer/src/lib.rs | 2 + .../src/serde_helpers/field_as_string.rs | 24 ++ .../src/serde_helpers/mod.rs | 1 + 8 files changed, 644 insertions(+), 32 deletions(-) create mode 100644 observability/crates/solana-price-indexer/src/jupiter.rs create mode 100644 observability/crates/solana-price-indexer/src/serde_helpers/field_as_string.rs create mode 100644 observability/crates/solana-price-indexer/src/serde_helpers/mod.rs diff --git a/observability/Cargo.lock b/observability/Cargo.lock index 6f9693e7..5a3e17ce 100644 --- a/observability/Cargo.lock +++ b/observability/Cargo.lock @@ -588,9 +588,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit", "memchr", @@ -614,8 +614,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -1719,6 +1719,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1892,7 +1907,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", "indexmap 2.2.6", "slab", "tokio", @@ -2044,6 +2078,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -2051,7 +2096,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -2083,9 +2151,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.25", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -2097,6 +2165,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2104,8 +2192,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.28", "rustls 0.21.10", "tokio", "tokio-rustls 0.24.1", @@ -2117,12 +2205,48 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.2.0", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2507,6 +2631,24 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.26.4" @@ -2764,12 +2906,50 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2 1.0.79", + "quote 1.0.35", + "syn 2.0.55", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_str_bytes" version = "6.6.1" @@ -3385,10 +3565,10 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.25", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-rustls", "ipnet", "js-sys", @@ -3416,6 +3596,48 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -3754,6 +3976,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_qs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c" +dependencies = [ + "percent-encoding", + "serde", + "thiserror", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4128,7 +4361,7 @@ dependencies = [ "gethostname", "lazy_static", "log", - "reqwest", + "reqwest 0.11.27", "solana-sdk", ] @@ -4273,7 +4506,7 @@ dependencies = [ "crossbeam-channel", "futures-util", "log", - "reqwest", + "reqwest 0.11.27", "semver", "serde", "serde_derive", @@ -4358,7 +4591,7 @@ dependencies = [ "bs58 0.4.0", "indicatif", "log", - "reqwest", + "reqwest 0.11.27", "semver", "serde", "serde_derive", @@ -4381,7 +4614,7 @@ dependencies = [ "base64 0.21.7", "bs58 0.4.0", "jsonrpc-core", - "reqwest", + "reqwest 0.11.27", "semver", "serde", "serde_derive", @@ -4664,9 +4897,14 @@ dependencies = [ name = "solana_price_indexer" version = "0.1.0" dependencies = [ + "anyhow", "backoff", "dotenv", "envconfig", + "futures", + "reqwest 0.12.2", + "serde", + "serde_qs", "thiserror", "tokio", "tracing", @@ -5252,6 +5490,16 @@ dependencies = [ "syn 2.0.55", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.23.4" @@ -5362,10 +5610,10 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.25", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -5446,6 +5694,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -5544,7 +5793,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes", - "http", + "http 0.2.12", "httparse", "log", "rand 0.8.5", @@ -6133,7 +6382,7 @@ source = "git+https://github.com/rpcpool/yellowstone-grpc.git?rev=a2cd1498ac64ba dependencies = [ "bytes", "futures", - "http", + "http 0.2.12", "thiserror", "tonic", "tonic-health", diff --git a/observability/crates/solana-price-indexer/Cargo.toml b/observability/crates/solana-price-indexer/Cargo.toml index 26dd7abb..dd02836d 100644 --- a/observability/crates/solana-price-indexer/Cargo.toml +++ b/observability/crates/solana-price-indexer/Cargo.toml @@ -8,9 +8,14 @@ name = "index-solana-prices" path = "bin/index_solana_prices.rs" [dependencies] +anyhow = "1.0.81" backoff = { workspace = true } dotenv = { workspace = true } envconfig = { workspace = true } +futures = { workspace = true } +reqwest = { version = "0.12.2", features = ["json"] } +serde = { workspace = true } +serde_qs = "0.12.0" thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/observability/crates/solana-price-indexer/bin/index_solana_prices.rs b/observability/crates/solana-price-indexer/bin/index_solana_prices.rs index ff16d54c..0d547058 100644 --- a/observability/crates/solana-price-indexer/bin/index_solana_prices.rs +++ b/observability/crates/solana-price-indexer/bin/index_solana_prices.rs @@ -50,7 +50,9 @@ pub async fn main() -> Result<(), IndexingError> { tracing::subscriber::set_global_default(subscriber).unwrap(); }; - let mut indexer = SolanaPriceIndexer::new(); + let mut indexer = SolanaPriceIndexer::new("https://quote-api.jup.ag/v6/".to_string()) + .await + .unwrap(); indexer.run().await; diff --git a/observability/crates/solana-price-indexer/src/indexer.rs b/observability/crates/solana-price-indexer/src/indexer.rs index 389ad936..ec0f9841 100644 --- a/observability/crates/solana-price-indexer/src/indexer.rs +++ b/observability/crates/solana-price-indexer/src/indexer.rs @@ -1,13 +1,201 @@ -pub struct SolanaPriceIndexer {} +use std::{collections::HashMap, time::Duration}; + +use anyhow::Result; +use futures::future::try_join_all; +use tokio::time::interval; +use tracing::warn; + +use crate::jupiter::{JupiterSwapApiClient, QuoteRequest, QuoteResponse}; + +/// TODOs/notes: +/// +/// - A single quote input amount might no be sufficient +/// - The quote amount is hardcoded, and does not accounts at all mint decimals or current $-value +/// - Concurrently fetching the price for all bank mints seems to be working without hitting Jup API rate limits, same for 2x the current amount of mints. This might not hold depending on how many input amounts we want to fetch. This might in turn affect how accurately the price datapoint gets taken. +/// - This solves live ingest, but not backfilling (required for the new token listing process) + +const USDC_MINT: &str = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v"; +const USDC_MINT_DECIMALS: u32 = 6; + +pub struct SolanaPriceIndexer { + jupiter_client: JupiterSwapApiClient, + tracked_mints: Vec, +} + +pub struct TrackedMint { + mint: String, + quote_request: QuoteRequest, + decimals: u32, +} + +#[derive(Debug, Clone)] +pub enum Price { + Price(f64), + Unavailable, +} impl SolanaPriceIndexer { - pub fn new() -> Self { - SolanaPriceIndexer {} + pub async fn new(jupitar_api_url: String) -> Result { + let jupiter_client = JupiterSwapApiClient::new(jupitar_api_url); + let mints_to_track = Self::fetch_mints_to_track().await?; + + let tracked_mints = mints_to_track + .iter() + .map(|(mint, decimals)| TrackedMint { + quote_request: QuoteRequest { + input_mint: mint.to_string(), + output_mint: USDC_MINT.to_string(), + amount: 1_000_000, + ..Default::default() + }, + decimals: *decimals, + mint: mint.to_string(), + }) + .collect(); + + Ok(SolanaPriceIndexer { + jupiter_client, + tracked_mints, + }) } pub async fn run(&mut self) { + let mut timer = interval(Duration::from_secs(10)); + timer.tick().await; loop { - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + let prices = self.fetch_prices().await.unwrap(); + println!("Prices: {:?}", prices); + timer.tick().await; } } + + async fn fetch_prices(&self) -> Result> { + let prices = try_join_all(self.tracked_mints.iter().map(|tm| async move { + let QuoteResponse { out_amount, .. } = + match self.jupiter_client.quote(&tm.quote_request).await { + Ok(response) => response, + Err(e) => { + if e.to_string().contains("Could not find any route") { + warn!("No route found for mint {}", tm.mint); + return anyhow::Ok((tm.mint.clone(), Price::Unavailable)); + } + panic!("Error fetching price for mint {}: {:?}", tm.mint, e); + } + }; + + let price = (out_amount as f64 / 10u64.pow(USDC_MINT_DECIMALS) as f64) + / (tm.quote_request.amount as f64 / 10u64.pow(tm.decimals) as f64); + + anyhow::Ok((tm.mint.clone(), Price::Price(price))) + })) + .await? + .iter() + .cloned() + .collect(); + + Ok(prices) + } + + // TODO: Make this fetched from a GCP bucket file or smth + async fn fetch_mints_to_track() -> Result> { + Ok(vec![ + ("So11111111111111111111111111111111111111112".to_string(), 9), + ( + "EKpQGSJtjMFqKZ9KQanSqYXRcF8fBopzLHYxdM65zcjm".to_string(), + 6, + ), + ( + "BLZEEuZUBVqFhj8adcCFPJvPVCiCyVmh3hkJMrU8KuJA".to_string(), + 9, + ), + ( + "DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263".to_string(), + 5, + ), + ("bSo13r4TkiE4KumL71LsHTPpL2euBYLFx6h9HP3piy1".to_string(), 9), + ( + "DUSTawucrTsGU8hcqRdHDCbuYhCPADMLM2VcCb8VnFnQ".to_string(), + 9, + ), + ( + "7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs".to_string(), + 8, + ), + ( + "AZsHEMXd36Bj1EMNXhowJajpUXzrKcK57wW4ZGXVa7yR".to_string(), + 5, + ), + ("hntyVP6YFm1Hg25TN9WGLqM12b8TQmcknKrdu1oxWux".to_string(), 8), + ( + "4vMsoUT2BWatFweudnQM1xedRLfJgJ7hswhcpz4xgBTy".to_string(), + 9, + ), + ( + "J1toso1uCk3RLmjorhTtrVwY9HJ7X8V9yYac6Y7kGCPn".to_string(), + 9, + ), + ( + "27G8MtK7VtTcCHkpASjSDdkWWYfoqT6ggEuKidVJidD4".to_string(), + 6, + ), + ("jtojtomepa8beP8AuQc6eXt5FriJwfFMwQx2v2f9mCL".to_string(), 9), + ("JUPyiwrYJFskUPiHa7hkeR8VUtAeFoSYbKedZNsDvCN".to_string(), 6), + ("kinXdEcpDQeHPEuQnqmUgtYykqKGVFq6CeVX5iAHJq6".to_string(), 5), + ("LFG1ezantSY2LPX8jRz2qa31pPEhpwN9msFDzZw4T9Q".to_string(), 7), + ("LSTxxxnJzKDFSLr4dUkPcmCf5VyryEqzPLz5j4bpxFp".to_string(), 9), + ("MNDEFzGvMt87ueuHvVU9VcTqsAP5b3fTGPsHuuPA5ey".to_string(), 9), + ("mb1eu7TzEc71KxDpsmsKoucSSuuoGLv1drys1oP2jh6".to_string(), 6), + ("mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So".to_string(), 9), + ( + "BqVHWpwUDgMik5gbTciFfozadpE2oZth5bxCDrgbDt52".to_string(), + 9, + ), + ("orcaEKTdK7LKz57vaAYr9QeNsVEPfiu6QeMU1kektZE".to_string(), 6), + ( + "HZ1JovNiVvGrGNiiYvEozEVgZ58xaU3RKwX8eACQBCt3".to_string(), + 6, + ), + ("rndrizKT3MK1iimdxRdWabcF7Zg7AR5T4nud4EkHBof".to_string(), 8), + ("RLBxxFkseAZ4RgJH3Sqn8jXxhmGoz9jWxDNJMh8pL7a".to_string(), 2), + ( + "7xKXtg2CW87d97TXJSDpbD5jBkheTqA83TZRuJosgAsU".to_string(), + 9, + ), + ("SHDWyBxihqiCj6YekG2GUr7wqKLeLAMK1gHZck9pL6y".to_string(), 9), + ("StepAscQoEioFxxWGnh2sLBDFp9d8rvKz2Yp39iDpyT".to_string(), 9), + ( + "7dHbWXmci3dT8UFYWYZweBLXgycu7Y3iL6trKn1Y7ARj".to_string(), + 9, + ), + ( + "6DNSN2BJsaPFdFFc1zP37kkeNe4Usc1Sqkzr9C9vPWcU".to_string(), + 8, + ), + ( + "Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB".to_string(), + 6, + ), + ( + "7kbnvuGBxxj8AG9qp8Scn56muWGaRaFqxg1FsRp3PaFT".to_string(), + 6, + ), + ( + "3NZ9JMVBmGAqocybic2c7LQCJScmgsAZ6vQqTDzcqmJh".to_string(), + 8, + ), + ("WENWENvqqNya429ubCdR81ZmD69brwQaaBYY6p3LCpk".to_string(), 5), + ("ZScHuTtqZukUrtZS43teTKGs2VqkKL8k4QCouR2n6Uo".to_string(), 8), + ("nosXBVoaCTtYdLvKY6Csb4AC8JCdQKKAaWYtx2ZMoo7".to_string(), 6), + ("SNSNkV9zfG5ZKWQs6x4hxvBRV6s8SqMfSGCtECDvdMd".to_string(), 9), + ("METADDFL6wWMWEoKTFJwcThTbUmtarRJZjRpzUvkxhr".to_string(), 9), + ( + "E1kvzJNxShvvWTrudokpzuc789vRiDXfXG3duCuY6ooE".to_string(), + 9, + ), + ( + "85VBFQZC9TZkfaptBWjvUw7YbZjy52A6mjtPGjstQAmQ".to_string(), + 6, + ), + ]) + } } diff --git a/observability/crates/solana-price-indexer/src/jupiter.rs b/observability/crates/solana-price-indexer/src/jupiter.rs new file mode 100644 index 00000000..f45b756c --- /dev/null +++ b/observability/crates/solana-price-indexer/src/jupiter.rs @@ -0,0 +1,141 @@ +use std::str::FromStr; + +use crate::serde_helpers::field_as_string; +use anyhow::{anyhow, Error, Result}; +use reqwest::{Client, Response}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Default, PartialEq, Clone, Debug)] +pub enum SwapMode { + #[default] + ExactIn, + ExactOut, +} + +impl FromStr for SwapMode { + type Err = Error; + + fn from_str(s: &str) -> std::result::Result { + match s { + "ExactIn" => Ok(Self::ExactIn), + "ExactOut" => Ok(Self::ExactOut), + _ => Err(anyhow!("{} is not a valid SwapMode", s)), + } + } +} + +#[derive(Serialize, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct QuoteRequest { + pub input_mint: String, + pub output_mint: String, + #[serde(with = "field_as_string")] + pub amount: u64, + pub swap_mode: Option, + pub slippage_bps: u16, + pub platform_fee_bps: Option, + pub dexes: Option>, + pub excluded_dexes: Option>, + pub only_direct_routes: Option, + pub as_legacy_transaction: Option, + pub max_accounts: Option, + pub quote_type: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct PlatformFee { + #[serde(with = "field_as_string")] + pub amount: u64, + pub fee_bps: u8, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct QuoteResponse { + pub input_mint: String, + #[serde(with = "field_as_string")] + pub in_amount: u64, + pub output_mint: String, + #[serde(with = "field_as_string")] + pub out_amount: u64, + /// Not used by build transaction + #[serde(with = "field_as_string")] + pub other_amount_threshold: u64, + pub swap_mode: SwapMode, + pub slippage_bps: u16, + pub platform_fee: Option, + pub price_impact_pct: String, + pub route_plan: RoutePlanWithMetadata, + #[serde(default)] + pub context_slot: u64, + #[serde(default)] + pub time_taken: f64, +} + +/// Topologically sorted DAG with additional metadata for rendering +pub type RoutePlanWithMetadata = Vec; + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +#[serde(rename_all = "camelCase")] +pub struct RoutePlanStep { + pub swap_info: SwapInfo, + pub percent: u8, +} + +#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SwapInfo { + pub amm_key: String, + pub label: String, + pub input_mint: String, + pub output_mint: String, + /// An estimation of the input amount into the AMM + #[serde(with = "field_as_string")] + pub in_amount: u64, + /// An estimation of the output amount into the AMM + #[serde(with = "field_as_string")] + pub out_amount: u64, + #[serde(with = "field_as_string")] + pub fee_amount: u64, + pub fee_mint: String, +} + +#[derive(Clone)] +pub struct JupiterSwapApiClient { + pub base_path: String, +} + +impl JupiterSwapApiClient { + pub fn new(base_path: String) -> Self { + Self { base_path } + } + + pub async fn quote(&self, quote_request: &QuoteRequest) -> Result { + let query = serde_qs::to_string("e_request)?; + let response = Client::new() + .get(format!("{}/quote?{query}", self.base_path)) + .send() + .await?; + check_status_code_and_deserialize(response).await + } +} + +async fn check_is_success(response: Response) -> Result { + if !response.status().is_success() { + return Err(anyhow!( + "Request status not ok: {}, body: {:?}", + response.status(), + response.text().await + )); + } + Ok(response) +} + +async fn check_status_code_and_deserialize(response: Response) -> Result { + check_is_success(response) + .await? + .json::() + .await + .map_err(Into::into) +} diff --git a/observability/crates/solana-price-indexer/src/lib.rs b/observability/crates/solana-price-indexer/src/lib.rs index 91f2f48b..1b87cc8d 100644 --- a/observability/crates/solana-price-indexer/src/lib.rs +++ b/observability/crates/solana-price-indexer/src/lib.rs @@ -1,2 +1,4 @@ pub mod error; pub mod indexer; +pub mod jupiter; +pub mod serde_helpers; diff --git a/observability/crates/solana-price-indexer/src/serde_helpers/field_as_string.rs b/observability/crates/solana-price-indexer/src/serde_helpers/field_as_string.rs new file mode 100644 index 00000000..467b5d66 --- /dev/null +++ b/observability/crates/solana-price-indexer/src/serde_helpers/field_as_string.rs @@ -0,0 +1,24 @@ +use { + serde::{de, Deserializer, Serializer}, + serde::{Deserialize, Serialize}, + std::str::FromStr, +}; + +pub fn serialize(t: &T, serializer: S) -> Result +where + T: ToString, + S: Serializer, +{ + t.to_string().serialize(serializer) +} + +pub fn deserialize<'de, T, D>(deserializer: D) -> Result +where + T: FromStr, + D: Deserializer<'de>, + ::Err: std::fmt::Debug, +{ + let s: String = String::deserialize(deserializer)?; + s.parse() + .map_err(|e| de::Error::custom(format!("Parse error: {:?}", e))) +} diff --git a/observability/crates/solana-price-indexer/src/serde_helpers/mod.rs b/observability/crates/solana-price-indexer/src/serde_helpers/mod.rs new file mode 100644 index 00000000..4ea7e47e --- /dev/null +++ b/observability/crates/solana-price-indexer/src/serde_helpers/mod.rs @@ -0,0 +1 @@ +pub mod field_as_string;