From f182fa5d0a77d7b4ce2cdf5ea93440afe1a11860 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Thu, 30 Apr 2026 07:10:02 +0000 Subject: [PATCH 1/2] Fix : Future year Timeouts in WorldBank IDS --- .../world_bank/worldbank_ids/download.py | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/statvar_imports/world_bank/worldbank_ids/download.py b/statvar_imports/world_bank/worldbank_ids/download.py index 55ab85d101..15805cd493 100644 --- a/statvar_imports/world_bank/worldbank_ids/download.py +++ b/statvar_imports/world_bank/worldbank_ids/download.py @@ -15,6 +15,7 @@ import os import sys import numpy as np +import pandas as pd from absl import logging, flags, app # Import the required function after installing the package from bblocks import WorldBankData, DebtIDS @@ -92,16 +93,37 @@ def main(_): # Itterating each list to download the respective data. - for idx, indicator in enumerate(indicator_list): + for idx, indicator_list_to_process in enumerate(indicator_list): # Creating an IDS object debt_id = DebtIDS() - # Load the indicators. - logging.info("Loaded data for%s", indicator) - debt_id.load_data(indicators=indicator, - start_year=FLAGS.start_year, - end_year=FLAGS.end_year) - # Get the data as a DataFrame - df = debt_id.get_data() + + all_data_frames = [] + + # Process indicators one by one to avoid crashing the whole batch + for ind in indicator_list_to_process: + try: + logging.info("Loading data for %s", ind) + debt_id.load_data(indicators=[ind], + start_year=FLAGS.start_year, + end_year=FLAGS.end_year) + # Get the data for this specific indicator + df_temp = debt_id.get_data() + if df_temp is not None and not df_temp.empty: + all_data_frames.append(df_temp) + + # Reset debt_id for next indicator if needed or use a new one + debt_id = DebtIDS() + except Exception as e: + logging.error("Failed to load data for %s: %s", ind, e) + continue + + if not all_data_frames: + logging.warning("No data loaded for category: %s", indicator_listname[idx]) + continue + + # Combine all successful indicators for this category + df = pd.concat(all_data_frames, ignore_index=True) + try: # Extracting only the year part for data. df["YEARMOD"]=df["year"].dt.year From d68b1645d60512a6fd8edc626013afe1e5efcb9e Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Sun, 3 May 2026 20:26:23 +0000 Subject: [PATCH 2/2] gemini-suggestion-fix --- statvar_imports/world_bank/worldbank_ids/download.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/statvar_imports/world_bank/worldbank_ids/download.py b/statvar_imports/world_bank/worldbank_ids/download.py index 15805cd493..1a3d9c386f 100644 --- a/statvar_imports/world_bank/worldbank_ids/download.py +++ b/statvar_imports/world_bank/worldbank_ids/download.py @@ -94,13 +94,11 @@ def main(_): # Itterating each list to download the respective data. for idx, indicator_list_to_process in enumerate(indicator_list): - # Creating an IDS object - debt_id = DebtIDS() - all_data_frames = [] # Process indicators one by one to avoid crashing the whole batch for ind in indicator_list_to_process: + debt_id = DebtIDS() try: logging.info("Loading data for %s", ind) debt_id.load_data(indicators=[ind], @@ -110,9 +108,6 @@ def main(_): df_temp = debt_id.get_data() if df_temp is not None and not df_temp.empty: all_data_frames.append(df_temp) - - # Reset debt_id for next indicator if needed or use a new one - debt_id = DebtIDS() except Exception as e: logging.error("Failed to load data for %s: %s", ind, e) continue