From edf8d8aa9afbf437e48017f7e9be950aabc5bad4 Mon Sep 17 00:00:00 2001
From: akmiller01 <alex.k.miller@gmail.com>
Date: Tue, 23 Apr 2024 10:02:38 -0400
Subject: [PATCH] New columns for CRS April 2024

---
 data_updates/Python/download_oecd.py | 17 ++++++++++-------
 data_updates/R/load_mirrors.R        |  4 ++++
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/data_updates/Python/download_oecd.py b/data_updates/Python/download_oecd.py
index 2aac802c2..67c69d674 100644
--- a/data_updates/Python/download_oecd.py
+++ b/data_updates/Python/download_oecd.py
@@ -146,13 +146,16 @@ def download(scrape_path, download_path, output_folder_prefix):
         # Unzip
         dir_path = os.path.dirname(os.path.realpath(__file__))
         remove_null_script_path = os.path.abspath(os.path.join(dir_path, "..", "remove_null.sh"))
-        with zipfile.ZipFile(path, "r") as zip_ref:
-            zip_ref.extractall(content_directory)
-            extracted_files = zip_ref.namelist()
-            for extracted_file in extracted_files:
-                full_path_extracted_file = os.path.join(content_directory, extracted_file)
-                rm_null_cmd = [remove_null_script_path, full_path_extracted_file]
-                subprocess.run(rm_null_cmd)
+        try:
+            with zipfile.ZipFile(path, "r") as zip_ref:
+                zip_ref.extractall(content_directory)
+                extracted_files = zip_ref.namelist()
+                for extracted_file in extracted_files:
+                    full_path_extracted_file = os.path.join(content_directory, extracted_file)
+                    rm_null_cmd = [remove_null_script_path, full_path_extracted_file]
+                    subprocess.run(rm_null_cmd)
+        except zipfile.BadZipFile:
+            print("{} is not a valid zip file. Skipping...".format(name))
 
     # Finished!
     print("Finished.\t\t\t")
diff --git a/data_updates/R/load_mirrors.R b/data_updates/R/load_mirrors.R
index 0b2670029..a0eae1979 100644
--- a/data_updates/R/load_mirrors.R
+++ b/data_updates/R/load_mirrors.R
@@ -160,6 +160,7 @@ merge_crs_tables = function(file_vec){
     "integer",
     "integer",
     "text",
+    "text",
     "integer",
     "text",
     "text",
@@ -167,6 +168,7 @@ merge_crs_tables = function(file_vec){
     "integer",
     "integer",
     "text",
+    "text",
     "integer",
     "text",
     "integer",
@@ -254,6 +256,7 @@ merge_crs_tables = function(file_vec){
   names(crs_field_types) = c(
     "year"
     ,"donor_code"
+    ,"donor_iso3_code" # New Apr 2024
     ,"donor_name"
     ,"agency_code"
     ,"agency_name"
@@ -261,6 +264,7 @@ merge_crs_tables = function(file_vec){
     ,"project_number"
     ,"initial_report"
     ,"recipient_code"
+    ,"recipient_iso3_code" # New Apr 2024
     ,"recipient_name"
     ,"region_code"
     ,"region_name"