feat: improve env variables

.env.example

@@ -8,13 +8,13 @@ REDIS_CONNECTION_CONFIG='{"redis_connection_str":"redis://127.0.0.1:6379/0"}'
 
 #RocksDB
 ROCKS_DB_PATH="/usr/src/rocksdb-data"
-ROCKS_DB_PATH_CONTAINER="/usr/src/rocksdb-data"
-ROCKS_DB_SECONDARY_PATH_CONTAINER="path/to/rocks/secondary/db"
+ROCKS_DB_SECONDARY_PATH="path/to/rocks/secondary/db"
 # path to the slots data, required for the backfiller to work
 ROCKS_SLOTS_DB_PATH=/path/to/slots-data
 ROCKS_SECONDARY_SLOTS_DB_PATH=/path/to/secondary/ingester-slots
 ROCKS_ARCHIVES_DIR="path/to/rocks/backup/archives"
 ROCKS_BACKUP_ARCHIVES_DIR="path/to/rocks/backup/archives"
+ROCKS_BACKUP_URL="127.0.0.1:3051/snapshot"
 ROCKS_MIGRATION_STORAGE_PATH=/path/to/migration_storage
 
 #Backfiller
@@ -77,9 +77,9 @@ SKIP_CHECK_TREE_GAPS=false
 
 # API_RPC_HOST  INGESTER_RPC_HOST -> RPC_HOST
 #INGESTER_ROCKS_DB_PATH -> ROCKS_DB_PATH
-#INGESTER_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH_CONTAINER
+#INGESTER_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH
 #INGESTER_SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
-#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH_CONTAINER
+#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH
 #INGESTER_FILE_STORAGE_PATH -> FILE_STORAGE_PATH
 #INGESTER_FILE_STORAGE_PATH_CONTAINER -> FILE_STORAGE_PATH_CONTAINER
 # INGESTER_PROFILING_FILE_PATH -> PROFILING_FILE_PATH
@@ -96,13 +96,13 @@ SKIP_CHECK_TREE_GAPS=false
 #API_DATABASE_CONFIG ->  PG_MAX_DB_CONNECTIONS  and  PG_DATABASE_URL
 
 #SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
-#SYNCHRONIZER_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH_CONTAINER
+#SYNCHRONIZER_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH
 #SYNCHRONIZER_DUMP_SYNCHRONIZER_BATCH_SIZE -> DUMP_SYNCHRONIZER_BATCH_SIZE
 #SYNCHRONIZER_DUMP_SYNC_THRESHOLD -> DUMP_SYNC_THRESHOLD
 
 #API_ARCHIVES_DIR -> rocks_archives_dir
-#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH_CONTAINER
-#API_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH_CONTAINER
+#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH
+#API_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH
 #API_FILE_STORAGE_PATH_CONTAINER -> FILE_STORAGE_PATH_CONTAINER
 #API_JSON_MIDDLEWARE_CONFIG -> JSON_MIDDLEWARE_CONFIG
 #API_CONSISTENCE_SYNCHRONIZATION_API_THRESHOLD - > CONSISTENCE_SYNCHRONIZATION_API_THRESHOLD

docker-compose.yaml

@@ -8,7 +8,7 @@ services:
       - .env
     network_mode: host
     volumes:
-      - ${ROCKS_DB_PATH}:${ROCKS_DB_PATH_CONTAINER}:rw
+      - ${ROCKS_DB_PATH}:${ROCKS_DB_PATH}:rw
       - ${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:rw
       - ${ROCKS_BACKUP_DIR}:${ROCKS_BACKUP_DIR}:rw
       - ${ROCKS_BACKUP_ARCHIVES_DIR}:${ROCKS_BACKUP_ARCHIVES_DIR}:rw
@@ -39,8 +39,8 @@ services:
       - .env
     network_mode: host
     volumes:
-      - ${ROCKS_DB_PATH_CONTAINER}:${ROCKS_DB_PATH_CONTAINER}:ro
-      - ${ROCKS_DB_SECONDARY_PATH_CONTAINER}/api:${ROCKS_DB_SECONDARY_PATH_CONTAINER}:rw
+      - ${ROCKS_DB_PATH}:${ROCKS_DB_PATH}:ro
+      - ${ROCKS_DB_SECONDARY_PATH}/api:${ROCKS_DB_SECONDARY_PATH}:rw
       - ${ROCKS_ARCHIVES_DIR}:${ROCKS_ARCHIVES_DIR}:ro
       - ${FILE_STORAGE_PATH}:${FILE_STORAGE_PATH_CONTAINER}:rw
       - ./heaps:/usr/src/app/heaps:rw
@@ -62,9 +62,9 @@ services:
       - .env
     network_mode: host
     volumes:
-      - ${ROCKS_DB_PATH}:${ROCKS_DB_PATH_CONTAINER}:rw
+      - ${ROCKS_DB_PATH}:${ROCKS_DB_PATH}:rw
       - ${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:rw
-      - ${ROCKS_DB_SECONDARY_PATH_CONTAINER}/synchronizer:${ROCKS_DB_SECONDARY_PATH_CONTAINER}:rw
+      - ${ROCKS_DB_SECONDARY_PATH}/synchronizer:${ROCKS_DB_SECONDARY_PATH}:rw
       - ./heaps:/usr/src/app/heaps:rw
       - ./migrations:/usr/src/app/migrations
     depends_on:
@@ -169,10 +169,10 @@ services:
       - .env
     network_mode: host
     volumes:
-      - ${ROCKS_DB_PATH}:${ROCKS_DB_PATH_CONTAINER}:ro
+      - ${ROCKS_DB_PATH}:${ROCKS_DB_PATH}:ro
       - ${ROCKS_BACKUP_DIR}:${ROCKS_BACKUP_DIR}:rw
       - ${ROCKS_BACKUP_ARCHIVES_DIR}:${ROCKS_BACKUP_ARCHIVES_DIR}:rw
-      - ${ROCKS_DB_SECONDARY_PATH}/backup:${ROCKS_DB_SECONDARY_PATH_CONTAINER}:rw
+      - ${ROCKS_DB_SECONDARY_PATH}/backup:${ROCKS_DB_SECONDARY_PATH}:rw
     build:
       context: .
       dockerfile: ingester.Dockerfile

nft_ingester/scripts/run_api.bash

@@ -1,8 +1,8 @@
 export RUST_LOG=info
 export RUST_BACKTRACE=1
 export API_DATABASE_CONFIG='{max_postgres_connections=250, url="postgres://solana:solana@localhost:5432/v3"}'
-export API_ROCKS_DB_PATH_CONTAINER="/rocksdb/rocksdb-data"
-export API_ROCKS_DB_SECONDARY_PATH_CONTAINER="/rocksdb/secondary-api-rocksdb-data"
+export API_ROCKS_DB_PATH="/rocksdb/rocksdb-data"
+export API_ROCKS_DB_SECONDARY_PATH="/rocksdb/secondary-api-rocksdb-data"
 export API_ROCKS_SYNC_INTERVAL_SECONDS=2
 export API_METRICS_PORT=8985
 export API_SERVER_PORT=8990

nft_ingester/src/api/dapi/asset.rs

@@ -219,6 +219,7 @@ pub async fn get_by_ids<
     let mut asset_selected_maps = rocks_db
         .get_asset_selected_maps_async(unique_asset_ids.clone(), owner_address, &options)
         .await?;
+
     if let Some(json_downloader) = json_downloader {
         let mut urls_to_download = Vec::new();
 

nft_ingester/src/bin/api/main.rs

@@ -69,7 +69,7 @@ pub async fn main() -> Result<(), IngesterError> {
     let mutexed_tasks = Arc::new(Mutex::new(tasks));
 
     let storage = Storage::open_secondary(
-        &args.rocks_db_path_container,
+        &args.rocks_db_path,
         &args.rocks_db_secondary_path,
         mutexed_tasks.clone(),
         red_metrics.clone(),

nft_ingester/src/bin/ingester/main.rs

@@ -121,7 +121,7 @@ pub async fn main() -> Result<(), IngesterError> {
                 ),
             )
             .expect("invalid rocks backup archives dir"),
-            &PathBuf::from_str(&args.rocks_db_path_container)
+            &PathBuf::from_str(&args.rocks_db_path)
                 .expect("invalid rocks backup archives dir"),
         )
         .await?;
@@ -133,7 +133,7 @@ pub async fn main() -> Result<(), IngesterError> {
     info!("Init primary storage...");
     let primary_rocks_storage = Arc::new(
         init_primary_storage(
-            &args.rocks_db_path_container,
+            &args.rocks_db_path,
             args.enable_rocks_migration.unwrap_or(false),
             &args.rocks_migration_storage_path,
             &metrics_state,

nft_ingester/src/bin/rocksdb_backup/main.rs

@@ -27,7 +27,7 @@ async fn main() -> Result<(), RocksDbBackupServiceError> {
     let mutexed_tasks = Arc::new(Mutex::new(tasks));
 
     let storage = Storage::open_secondary(
-        &args.rocks_db_path_container,
+        &args.rocks_db_path,
         &args.rocks_db_secondary_path,
         mutexed_tasks.clone(),
         red_metrics.clone(),
@@ -36,7 +36,7 @@ async fn main() -> Result<(), RocksDbBackupServiceError> {
     .unwrap();
 
     debug!(
-        rocks_db_path_container = ?args.rocks_db_path_container,
+        rocks_db_path_container = ?args.rocks_db_path,
         rocks_db_secondary_path = ?args.rocks_db_secondary_path,
         "Opened RocksDb in secondary mode"
     );

nft_ingester/src/bin/synchronizer/main.rs

@@ -60,7 +60,7 @@ pub async fn main() -> Result<(), IngesterError> {
     let mutexed_tasks = Arc::new(Mutex::new(tasks));
 
     let storage = Storage::open_secondary(
-        &args.rocks_db_path_container,
+        &args.rocks_db_path,
         &args.rocks_db_secondary_path,
         mutexed_tasks.clone(),
         red_metrics.clone(),

nft_ingester/src/config.rs

@@ -27,7 +27,7 @@ pub struct IngesterClapArgs {
         default_value = "./my_rocksdb",
         help = "Rocks db path container"
     )]
-    pub rocks_db_path_container: String,
+    pub rocks_db_path: String,
 
     #[clap(
         short('f'),
@@ -284,7 +284,7 @@ pub struct SynchronizerClapArgs {
         default_value = "./my_rocksdb",
         help = "Rocks db path container"
     )]
-    pub rocks_db_path_container: String,
+    pub rocks_db_path: String,
     #[clap(
         long,
         env,
@@ -335,7 +335,7 @@ pub struct SynchronizerClapArgs {
 #[command(author, version, about, long_about = None)]
 pub struct RocksDbBackupServiceClapArgs {
     #[clap(long, env, default_value = "./my_rocksdb", help = "Rocks db path container")]
-    pub rocks_db_path_container: PathBuf,
+    pub rocks_db_path: PathBuf,
     #[clap(long, env, default_value = "./my_rocksdb_secondary", help = "Rocks db secondary path")]
     pub rocks_db_secondary_path: PathBuf,
     #[clap(long, env = "ROCKS_BACKUP_ARCHIVES_DIR", help = "Rocks backup archives dir")]
@@ -419,7 +419,7 @@ pub struct ApiClapArgs {
         default_value = "./my_rocksdb",
         help = "Rocks db path container"
     )]
-    pub rocks_db_path_container: String,
+    pub rocks_db_path: String,
     #[clap(
         long,
         env,
@@ -615,7 +615,7 @@ mod tests {
             "{}",
         ]);
 
-        assert_eq!(args.rocks_db_path_container, "./my_rocksdb");
+        assert_eq!(args.rocks_db_path, "./my_rocksdb");
         assert_eq!(args.file_storage_path_container, "./tmp/file_storage");
         assert_eq!(args.pg_max_db_connections, 100);
         assert_eq!(args.sequence_consistent_checker_wait_period_sec, 60);
@@ -647,7 +647,7 @@ mod tests {
 
         assert_eq!(args.rocks_dump_path, "./tmp/rocks_dump");
         assert_eq!(args.pg_max_db_connections, 100);
-        assert_eq!(args.rocks_db_path_container, "./my_rocksdb");
+        assert_eq!(args.rocks_db_path, "./my_rocksdb");
         assert_eq!(args.rocks_db_secondary_path, "./my_rocksdb_secondary");
         assert_eq!(args.run_profiling, false);
         assert_eq!(args.heap_path, "/usr/src/app/heaps");
@@ -668,7 +668,7 @@ mod tests {
             "postgres://solana:solana@localhost:5432/aura_db",
         ]);
 
-        assert_eq!(args.rocks_db_path_container, "./my_rocksdb");
+        assert_eq!(args.rocks_db_path, "./my_rocksdb");
         assert_eq!(args.rocks_db_secondary_path, "./my_rocksdb_secondary");
         assert_eq!(args.rocks_sync_interval_seconds, 2);
         assert_eq!(args.heap_path, "/usr/src/app/heaps");