query cost optimalization

Author: mr-arianelabs

hedera-balances/native.py

@@ -14,8 +14,19 @@ def get_global_start(client, project, dataset, max_batch_size):
 
   logger.info(f"Looking for latest ingested balance")
   # Perform a query.
+  latest_balance_partition_query = """
+    SELECT PARSE_TIMESTAMP("%Y%m%d", MAX(partition_id)) AS latest_partition
+    FROM `{project}.{dataset}.INFORMATION_SCHEMA.PARTITIONS`
+    WHERE table_name = "native_token_balance" AND partition_id != "__NULL__"
+  """
+  query_job = client.query(latest_balance_partition_query)  # API request
+  rows = query_job.result()  # Waits for query to finish
+
+  row = next(rows)
+  latest_partition = row.latest_partition
+
   latest_balance_query = f"""
-    SELECT MAX(created) AS latest FROM `{project}.{dataset}.native_token_balance` WHERE created >= "2019-01-01"
+    SELECT MAX(created) AS latest FROM `{project}.{dataset}.native_token_balance` WHERE created >= "{latest_partition}"
     """
 
   query_job = client.query(latest_balance_query)  # API request

scripts/balances-loop.sh

@@ -3,7 +3,7 @@
 set -e
 set -x
 
-START_DATE="2025-08-20"
+START_DATE="2025-09-09"
 INCREMENT="+1 day"
 
 PROJECT_ID="hedera-etl-bq"
@@ -15,11 +15,15 @@ export GOOGLE_CLOUD_PROJECT="$PROJECT_ID"
 
 ## MAIN PROGRAM
 while true; do
+    if [ -e stop_balances_loop ]; then
+        echo "Poison pill detected. Good bye"
+        exit 0
+    fi
     LATEST_TABLE="hedera_technical.balance_latest_$(date -d "${START_DATE} -1 day" +"%Y%m%d")"
     echo "Running balances for date ${START_DATE}"
-    WINDOW_IN_MINUTES=5
-    POLLING_WAIT=1
-    python3 -m hedera-balances -p hedera-etl-bq -d hedera_restricted -x hedera_technical -w $POLLING_WAIT -m $WINDOW_IN_MINUTES -t native -e START_DATE -l $LATEST_TABLE
+    WINDOW_IN_MINUTES=10
+    POLLING_WAIT=300
+    python3 -m hedera-balances -p hedera-etl-bq -d hedera_restricted -x hedera_technical -w $POLLING_WAIT -m $WINDOW_IN_MINUTES -t native -e $START_DATE -l $LATEST_TABLE
     START_DATE=$(date -d "${START_DATE} +1 day" --iso)
     sleep 10s
 done