diff --git a/Makefile b/Makefile index 5522c2d63..ee8d29c67 100644 --- a/Makefile +++ b/Makefile @@ -143,6 +143,7 @@ check-tools: ## Checks if development tools are installed @command -v taplo >/dev/null 2>&1 && echo "[OK] taplo is installed" || echo "[MISSING] taplo (make install-tools)" @command -v cargo-machete >/dev/null 2>&1 && echo "[OK] cargo-machete is installed" || echo "[MISSING] cargo-machete (make install-tools)" @command -v npm >/dev/null 2>&1 && echo "[OK] npm is installed" || echo "[MISSING] npm is not installed (run: make install-tools)" + @command -v diesel >/dev/null 2>&1 && echo "[OK] diesel is installed" || echo "[MISSING] diesel (make install-tools)" .PHONY: install-tools install-tools: ## Installs tools required by the Makefile @@ -153,6 +154,7 @@ install-tools: ## Installs tools required by the Makefile cargo install cargo-nextest --locked cargo install taplo-cli --locked cargo install cargo-machete --locked + cargo install diesel_cli --no-default-features --features sqlite --locked @if ! command -v node >/dev/null 2>&1; then \ echo "Node.js not found. Please install Node.js from https://nodejs.org/ or using your package manager"; \ echo "On macOS: brew install node"; \ @@ -161,3 +163,15 @@ install-tools: ## Installs tools required by the Makefile exit 1; \ fi @echo "Development tools installation complete!" + +# --- database schema --------------------------------------------------------------------------------- + +.PHONY: schema +schema: ## Regenerates crates/store/src/db/schema.rs from migrations + @command -v diesel >/dev/null 2>&1 || { echo "diesel CLI not found. Run: make install-tools"; exit 1; } + @echo "Regenerating Diesel schema..." + @rm -f /tmp/miden-schema.db + @diesel migration run --database-url=/tmp/miden-schema.db --migration-dir=crates/store/src/db/migrations --config-file=crates/store/diesel.toml + @diesel print-schema --database-url=/tmp/miden-schema.db --config-file=crates/store/diesel.toml --patch-file=crates/store/schema.patch > crates/store/src/db/schema.rs + @rm -f /tmp/miden-schema.db + @echo "Schema regenerated: crates/store/src/db/schema.rs" diff --git a/crates/store/schema.patch b/crates/store/schema.patch new file mode 100644 index 000000000..61610830b --- /dev/null +++ b/crates/store/schema.patch @@ -0,0 +1,122 @@ +--- /tmp/schema_base.rs 2026-01-12 15:09:35.714958597 +0100 ++++ crates/store/src/db/schema.rs 2026-01-12 15:14:12.430988158 +0100 +@@ -7,48 +7,48 @@ + } + } + + diesel::table! { + account_storage_map_values (account_id, block_num, slot_name, key) { + account_id -> Binary, +- block_num -> Integer, ++ block_num -> BigInt, + slot_name -> Text, + key -> Binary, + value -> Binary, + is_latest -> Bool, + } + } + + diesel::table! { + account_vault_assets (account_id, block_num, vault_key) { + account_id -> Binary, +- block_num -> Integer, ++ block_num -> BigInt, + vault_key -> Binary, + asset -> Nullable, + is_latest -> Bool, + } + } + + diesel::table! { + accounts (account_id, block_num) { + account_id -> Binary, +- network_account_id_prefix -> Nullable, +- block_num -> Integer, ++ network_account_id_prefix -> Nullable, ++ block_num -> BigInt, + account_commitment -> Binary, + code_commitment -> Nullable, +- nonce -> Nullable, ++ nonce -> Nullable, + storage_header -> Nullable, + vault_root -> Nullable, + is_latest -> Bool, +- created_at_block -> Integer, ++ created_at_block -> BigInt, + } + } + + diesel::table! { + block_headers (block_num) { +- block_num -> Integer, ++ block_num -> BigInt, + block_header -> Binary, + } + } + + diesel::table! { + note_scripts (script_root) { +@@ -56,54 +56,56 @@ + script -> Binary, + } + } + + diesel::table! { + notes (committed_at, batch_index, note_index) { +- committed_at -> Integer, ++ committed_at -> BigInt, + batch_index -> Integer, + note_index -> Integer, + note_id -> Binary, + note_commitment -> Binary, + note_type -> Integer, + sender -> Binary, + tag -> Integer, + execution_mode -> Integer, +- aux -> Integer, +- execution_hint -> Integer, ++ aux -> BigInt, ++ execution_hint -> BigInt, + inclusion_path -> Binary, +- consumed_at -> Nullable, ++ consumed_at -> Nullable, + nullifier -> Nullable, + assets -> Nullable, + inputs -> Nullable, + script_root -> Nullable, + serial_num -> Nullable, + } + } + + diesel::table! { + nullifiers (nullifier) { + nullifier -> Binary, + nullifier_prefix -> Integer, +- block_num -> Integer, ++ block_num -> BigInt, + } + } + + diesel::table! { + transactions (transaction_id) { + transaction_id -> Binary, + account_id -> Binary, +- block_num -> Integer, ++ block_num -> BigInt, + initial_state_commitment -> Binary, + final_state_commitment -> Binary, + nullifiers -> Binary, + output_notes -> Binary, +- size_in_bytes -> Integer, ++ size_in_bytes -> BigInt, + } + } + ++diesel::joinable!(accounts -> account_codes (code_commitment)); ++ + diesel::allow_tables_to_appear_in_same_query!( + account_codes, + account_storage_map_values, + account_vault_assets, + accounts, + block_headers, diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index 6bf6af3cf..9395cad63 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -1,5 +1,12 @@ // @generated automatically by Diesel CLI. +diesel::table! { + account_codes (code_commitment) { + code_commitment -> Binary, + code -> Binary, + } +} + diesel::table! { account_storage_map_values (account_id, block_num, slot_name, key) { account_id -> Binary, @@ -25,24 +32,17 @@ diesel::table! { accounts (account_id, block_num) { account_id -> Binary, network_account_id_prefix -> Nullable, + block_num -> BigInt, account_commitment -> Binary, code_commitment -> Nullable, nonce -> Nullable, storage_header -> Nullable, vault_root -> Nullable, - block_num -> BigInt, is_latest -> Bool, created_at_block -> BigInt, } } -diesel::table! { - account_codes (code_commitment) { - code_commitment -> Binary, - code -> Binary, - } -} - diesel::table! { block_headers (block_num) { block_num -> BigInt, @@ -102,20 +102,12 @@ diesel::table! { } diesel::joinable!(accounts -> account_codes (code_commitment)); -diesel::joinable!(accounts -> block_headers (block_num)); -// Note: Cannot use diesel::joinable! with accounts table due to composite primary key -// diesel::joinable!(notes -> accounts (sender)); -// diesel::joinable!(transactions -> accounts (account_id)); -diesel::joinable!(notes -> block_headers (committed_at)); -diesel::joinable!(notes -> note_scripts (script_root)); -diesel::joinable!(nullifiers -> block_headers (block_num)); -diesel::joinable!(transactions -> block_headers (block_num)); diesel::allow_tables_to_appear_in_same_query!( account_codes, account_storage_map_values, - accounts, account_vault_assets, + accounts, block_headers, note_scripts, notes, diff --git a/docs/internal/src/store.md b/docs/internal/src/store.md index 1929b7c49..5f59396f8 100644 --- a/docs/internal/src/store.md +++ b/docs/internal/src/store.md @@ -14,6 +14,35 @@ Note that the migration logic includes both a schema number _and_ a hash based o on node startup to ensure that any existing database matches the expected schema. If you're seeing database failures on startup its likely that you created the database _before_ making schema changes resulting in different schema hashes. +### Regenerating the Schema + +After modifying SQL migrations in `crates/store/src/db/migrations/`, regenerate the Diesel schema: + +```sh +make schema +``` + +This runs the migrations against an ephemeral SQLite database and generates `crates/store/src/db/schema.rs`. + +A patch file (`crates/store/schema.patch`) applies customizations to the generated schema: + +- **`BigInt` type mappings**: SQLite `INTEGER` columns map to Diesel's `Integer` (i32) by default, but our code uses `i64` for block numbers, nonces, timestamps, etc. The patch changes these to `BigInt`. +- **`joinable!` macro**: Adds `accounts -> account_codes` relationship for implicit joins. + +#### Updating the Patch + +When adding new columns that need `i64` in Rust: + +1. Run `make schema` to regenerate with the current patch +2. Edit `schema.rs` to change `Integer` to `BigInt` for the new column +3. Regenerate the patch: + ```sh + diesel migration run --database-url=/tmp/miden.db --migration-dir=crates/store/src/db/migrations --config-file=crates/store/diesel.toml + diesel print-schema --database-url=/tmp/miden.db --config-file=crates/store/diesel.toml > /tmp/base.rs + diff -U6 /tmp/base.rs crates/store/src/db/schema.rs > crates/store/schema.patch + rm /tmp/miden.db /tmp/base.rs + ``` + ## Architecture The store consists mainly of a gRPC server which answers requests from the RPC and block-producer components, as well as