branched uploading to aws

ecohealthalliance · Oct 13, 2023 · 48b2f59 · 48b2f59
1 parent 1e37b89
commit 48b2f59
Show file tree

Hide file tree

Showing 3 changed files with 1,199 additions and 571 deletions.
diff --git a/R/transform_modis_ndvi.R b/R/transform_modis_ndvi.R
@@ -3,7 +3,7 @@
 #' .. content for \details{} ..
 #'
 #' @title
-#' @param modis_ndvi_downloaded
+#' @param modis_ndvi_downloaded_subset
 #' @param continent_raster_template
 #' @param modis_ndvi_directory_transformed
 #' @param overwrite
@@ -33,7 +33,7 @@ transform_modis_ndvi <- function(modis_ndvi_downloaded_subset,
   }
 
   # Transform with template raster
-  transformed_raster <- transform_raster(raw_raster = rast(modis_ndvi_downloaded),
+  transformed_raster <- transform_raster(raw_raster = rast(modis_ndvi_downloaded_subset),
                                          template = rast(continent_raster_template))
 
   # Convert to dataframe

diff --git a/_targets.R b/_targets.R
@@ -96,12 +96,13 @@ dynamic_targets <- tar_plan(
              repository = "local"), 
 
   # save transformed to AWS bucket
-  tar_target(sentinel_ndvi_transformed_upload_aws_s3,  {sentinel_ndvi_transformed; # enforce dependency
-    aws_s3_upload_single_type(directory_path = sentinel_ndvi_directory_transformed,
-                              bucket =  aws_bucket,
-                              key = sentinel_ndvi_directory_transformed, 
-                              check = TRUE)}, 
-    cue = tar_cue("never")), 
+  tar_target(sentinel_ndvi_transformed_upload_aws_s3, 
+             aws_s3_upload(path = sentinel_ndvi_transformed,
+                           bucket =  aws_bucket,
+                           key = sentinel_ndvi_transformed, 
+                           check = TRUE), 
+             pattern = sentinel_ndvi_transformed,
+             cue = tar_cue("never")),    
 
   # MODIS NDVI -----------------------------------------------------------
   # 2005-present
@@ -166,12 +167,13 @@ dynamic_targets <- tar_plan(
              repository = "local"), 
 
   # save transformed to AWS bucket
-  tar_target(modis_ndvi_transformed_upload_aws_s3,  {modis_ndvi_transformed; # enforce dependency
-    aws_s3_upload_single_type(directory_path = modis_ndvi_directory_transformed,
-                              bucket =  aws_bucket,
-                              key = modis_ndvi_directory_transformed, 
-                              check = TRUE)}, 
-    cue = tar_cue("never")), 
+  tar_target(modis_ndvi_transformed_upload_aws_s3,
+             aws_s3_upload(directory_path = modis_ndvi_transformed,
+                           bucket =  aws_bucket,
+                           key = modis_ndvi_transformed, 
+                           check = TRUE), 
+             pattern = modis_ndvi_transformed,
+             cue = tar_cue("never")), 
 
   # NASA POWER recorded weather -----------------------------------------------------------
   # RH2M            MERRA-2 Relative Humidity at 2 Meters (%) ;
@@ -218,10 +220,10 @@ dynamic_targets <- tar_plan(
 
   # project to the template and save as arrow dataset
   tar_target(nasa_weather_transformed, 
-             tranform_nasa_weather(nasa_weather_pre_transformed,
-                                   nasa_weather_directory_transformed, 
-                                   continent_raster_template,
-                                   overwrite = FALSE),
+             transform_nasa_weather(nasa_weather_pre_transformed,
+                                    nasa_weather_directory_transformed, 
+                                    continent_raster_template,
+                                    overwrite = FALSE),
              pattern = nasa_weather_pre_transformed,
              format = "file", 
              repository = "local"),  
@@ -233,7 +235,7 @@ dynamic_targets <- tar_plan(
                            key = nasa_weather_transformed,
                            check = TRUE), 
              pattern = nasa_weather_transformed,
-             cue = tar_cue("thorough")),    
+             cue = tar_cue("never")),    
 
   # ECMWF Weather Forecast data -----------------------------------------------------------
   tar_target(ecmwf_forecasts_directory_raw, 
@@ -276,12 +278,13 @@ dynamic_targets <- tar_plan(
              repository = "local"),  
 
   # save transformed to AWS bucket
-  tar_target(ecmwf_forecasts_transformed_upload_aws_s3,  {ecmwf_forecasts_transformed; # enforce dependency
-    aws_s3_upload_single_type(directory_path = ecmwf_forecasts_directory_transformed,
-                              bucket =  aws_bucket,
-                              key = ecmwf_forecasts_directory_transformed, 
-                              check = TRUE)}, 
-    cue = tar_cue("never")),   
+  tar_target(ecmwf_forecasts_transformed_upload_aws_s3,  # enforce dependency
+             aws_s3_upload(directory_path = ecmwf_forecasts_transformed,
+                           bucket =  aws_bucket,
+                           key = ecmwf_forecasts_transformed, 
+                           check = TRUE), 
+             pattern = ecmwf_forecasts_transformed,
+             cue = tar_cue("never")),   
 
 )