New columns for CRS April 2024

devinit · Apr 23, 2024 · edf8d8a · edf8d8a
1 parent 44145fc
commit edf8d8a
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 7 deletions.
diff --git a/data_updates/Python/download_oecd.py b/data_updates/Python/download_oecd.py
@@ -146,13 +146,16 @@ def download(scrape_path, download_path, output_folder_prefix):
         # Unzip
         dir_path = os.path.dirname(os.path.realpath(__file__))
         remove_null_script_path = os.path.abspath(os.path.join(dir_path, "..", "remove_null.sh"))
-        with zipfile.ZipFile(path, "r") as zip_ref:
-            zip_ref.extractall(content_directory)
-            extracted_files = zip_ref.namelist()
-            for extracted_file in extracted_files:
-                full_path_extracted_file = os.path.join(content_directory, extracted_file)
-                rm_null_cmd = [remove_null_script_path, full_path_extracted_file]
-                subprocess.run(rm_null_cmd)
+        try:
+            with zipfile.ZipFile(path, "r") as zip_ref:
+                zip_ref.extractall(content_directory)
+                extracted_files = zip_ref.namelist()
+                for extracted_file in extracted_files:
+                    full_path_extracted_file = os.path.join(content_directory, extracted_file)
+                    rm_null_cmd = [remove_null_script_path, full_path_extracted_file]
+                    subprocess.run(rm_null_cmd)
+        except zipfile.BadZipFile:
+            print("{} is not a valid zip file. Skipping...".format(name))
 
     # Finished!
     print("Finished.\t\t\t")

diff --git a/data_updates/R/load_mirrors.R b/data_updates/R/load_mirrors.R
@@ -160,13 +160,15 @@ merge_crs_tables = function(file_vec){
     "integer",
     "integer",
     "text",
+    "text",
     "integer",
     "text",
     "text",
     "text",
     "integer",
     "integer",
     "text",
+    "text",
     "integer",
     "text",
     "integer",
@@ -254,13 +256,15 @@ merge_crs_tables = function(file_vec){
   names(crs_field_types) = c(
     "year"
     ,"donor_code"
+    ,"donor_iso3_code" # New Apr 2024
     ,"donor_name"
     ,"agency_code"
     ,"agency_name"
     ,"crs_id"
     ,"project_number"
     ,"initial_report"
     ,"recipient_code"
+    ,"recipient_iso3_code" # New Apr 2024
     ,"recipient_name"
     ,"region_code"
     ,"region_name"