diff --git a/statvar_imports/world_bank/worldbank_ids/download.py b/statvar_imports/world_bank/worldbank_ids/download.py index 55ab85d101..1a3d9c386f 100644 --- a/statvar_imports/world_bank/worldbank_ids/download.py +++ b/statvar_imports/world_bank/worldbank_ids/download.py @@ -15,6 +15,7 @@ import os import sys import numpy as np +import pandas as pd from absl import logging, flags, app # Import the required function after installing the package from bblocks import WorldBankData, DebtIDS @@ -92,16 +93,32 @@ def main(_): # Itterating each list to download the respective data. - for idx, indicator in enumerate(indicator_list): - # Creating an IDS object - debt_id = DebtIDS() - # Load the indicators. - logging.info("Loaded data for%s", indicator) - debt_id.load_data(indicators=indicator, - start_year=FLAGS.start_year, - end_year=FLAGS.end_year) - # Get the data as a DataFrame - df = debt_id.get_data() + for idx, indicator_list_to_process in enumerate(indicator_list): + all_data_frames = [] + + # Process indicators one by one to avoid crashing the whole batch + for ind in indicator_list_to_process: + debt_id = DebtIDS() + try: + logging.info("Loading data for %s", ind) + debt_id.load_data(indicators=[ind], + start_year=FLAGS.start_year, + end_year=FLAGS.end_year) + # Get the data for this specific indicator + df_temp = debt_id.get_data() + if df_temp is not None and not df_temp.empty: + all_data_frames.append(df_temp) + except Exception as e: + logging.error("Failed to load data for %s: %s", ind, e) + continue + + if not all_data_frames: + logging.warning("No data loaded for category: %s", indicator_listname[idx]) + continue + + # Combine all successful indicators for this category + df = pd.concat(all_data_frames, ignore_index=True) + try: # Extracting only the year part for data. df["YEARMOD"]=df["year"].dt.year