Merge branch 'main' of https://github.com/eco4cast/neon4cast-ci

scoring/scoring.R

@@ -2,6 +2,7 @@ library(score4cast)
 library(arrow)
 
 past_days <- 365
+cut_off_date <- lubridate::as_date("2023-12-31")
 n_cores <- 8
 
 setwd(here::here())
@@ -33,20 +34,21 @@ variable_duration <- arrow::open_dataset(s3_inv) |>
   dplyr::distinct(variable, duration, project_id) |>
   dplyr::collect()
 
-future::plan("future::multisession", workers = n_cores)
+#future::plan("future::multisession", workers = n_cores)
 
-#future::plan("future::sequential", workers = n_cores)
+future::plan("future::sequential")
 
 furrr::future_walk(1:nrow(variable_duration), function(k, variable_duration, config, endpoint){
 
   Sys.setenv(AWS_ACCESS_KEY_ID=Sys.getenv("OSN_KEY"),
              AWS_SECRET_ACCESS_KEY=Sys.getenv("OSN_SECRET"))
 
+
   variable <- variable_duration$variable[k]
   duration <- variable_duration$duration[k]
   project_id <- variable_duration$project_id[k]
 
-  print(variable_duration[k,])
+  #print(variable_duration[k,])
 
   s3_targets <- arrow::s3_bucket(glue::glue(config$targets_bucket,"/project_id={project_id}"), endpoint_override = endpoint)
   s3_scores <- arrow::s3_bucket(config$scores_bucket, endpoint_override = endpoint)
@@ -95,7 +97,8 @@ furrr::future_walk(1:nrow(variable_duration), function(k, variable_duration, con
     dplyr::collect() |>
     dplyr::distinct() |>
     dplyr::filter(date > Sys.Date() - lubridate::days(past_days),
-                  date <= lubridate::as_date(max(target$datetime))) |>
+                  date <= lubridate::as_date(max(target$datetime)),
+                  ((date > cut_off_date & duration %in% c("P1D","PT30M")) | !(duration %in% c("P1D","PT30M")))) |>
     dplyr::group_by(model_id, date, duration, path, endpoint) |>
     dplyr::arrange(reference_date, pub_date) |>
     dplyr::summarise(reference_date = paste(unique(reference_date), collapse=","),
@@ -106,6 +109,9 @@ furrr::future_walk(1:nrow(variable_duration), function(k, variable_duration, con
 
     new_prov <- purrr::map_dfr(1:nrow(groupings), function(j, groupings, prov_df, s3_scores_path, curr_variable){
 
+
+      print(c(variable_duration[k,], j))
+
       group <- groupings[j,]
       ref <- group$date
 
@@ -118,7 +124,8 @@ furrr::future_walk(1:nrow(variable_duration), function(k, variable_duration, con
 
       if (!(score4cast:::prov_has(id, prov_df, "new_id"))){
 
-        print(group)
+        print(paste0("s3://anonymous@",group$path,"/model_id=",group$model_id,"?endpoint_override=",group$endpoint))
+        print(group$reference_date)
 
 
         reference_dates <- unlist(stringr::str_split(group$reference_date, ","))