Skip to content

Commit

Permalink
Use post_process to zip files after evaluation
Browse files Browse the repository at this point in the history
  • Loading branch information
henrykironde committed Dec 14, 2023
1 parent 5220f6f commit 0d42c4c
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 23 deletions.
1 change: 1 addition & 0 deletions NAMESPACE
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ export(portal_ndvi_href)
export(portal_project_href)
export(portal_weather_href)
export(portalcast)
export(post_process)
export(prefab_datasets)
export(prefab_datasets_controls)
export(prefab_models)
Expand Down
16 changes: 10 additions & 6 deletions R/fill_dir.R
Original file line number Diff line number Diff line change
Expand Up @@ -242,18 +242,12 @@ fill_forecasts <- function (main = ".") {

messageq(paste0(" Located ", length(files), " forecast file(s) in resources to be moved to directory ..."), quiet = settings$quiet)
messageq(" ... moving ... ", quiet = settings$quiet)

# zip the forecast files before copying them back
zip_unzip("zip", forecast_path = forecasts_path(main = main))

copied <- file.copy(from = files,
to = forecasts_path(main = main),
recursive = TRUE)

messageq(paste0(" ... ", sum(copied), " files moved. "), quiet = settings$quiet)

# zip the forecast zipped files
zip_unzip("zip", forecast_path = forecasts_path(main = main))
invisible( )

}
Expand Down Expand Up @@ -319,3 +313,13 @@ fill_models <- function (main = ".",
}


#' After portalcast and evaluate_forecast run post_process to zip files
#'
#' @param main location of main directory with a forecast dir
#'
#' @export
#'
post_process <- function(main= ".") {
settings <- read_directory_settings(main = main)
zip_unzip("zip", forecasts_path(main = main))

Check warning on line 324 in R/fill_dir.R

View check run for this annotation

Codecov / codecov/patch

R/fill_dir.R#L323-L324

Added lines #L323 - L324 were not covered by tests
}
39 changes: 22 additions & 17 deletions R/zip_unzip_forecasts.R
Original file line number Diff line number Diff line change
Expand Up @@ -20,68 +20,73 @@ zip_unzip <- function(type = NULL,
utils::zip(zipfile, files)
unlink(files)
}

messageq("Preparing forecasts files")
proj_path <- forecast_path
forecasts_metadata <- paste0(proj_path, "/forecasts_metadata.csv")

proj_path <- normalizePath(proj_path, mustWork = FALSE)
forecasts_metadata <- normalizePath(forecasts_metadata, mustWork = FALSE)
forecasts_metadata <-
normalizePath(forecasts_metadata, mustWork = FALSE)
metadata <- read.csv(forecasts_metadata)
unique_dates <- sort(unique(metadata$"forecast_date"))

if (type == "zip") {
csv_file <- "_forecast_table.csv"
yaml_file <- "_metadata.yaml"
json_file <- "_model_forecast.json"

if (!is.null(date)) {
unique_dates <- c(date)
messageq(paste0("Zipping forecasts files for ", date))
}

for (forecast_day in unique_dates) {
id_date_files <- c()
zipfile <- paste0(proj_path, "/forecast_id_", forecast_day, ".zip")

zipfile <-
paste0(proj_path, "/forecast_id_", forecast_day, ".zip")

Check warning on line 47 in R/zip_unzip_forecasts.R

View check run for this annotation

Codecov / codecov/patch

R/zip_unzip_forecasts.R#L46-L47

Added lines #L46 - L47 were not covered by tests

# Get all the values of that particular day in a data frame
newdata <- subset(metadata,
newdata <- subset(
metadata,

Check warning on line 51 in R/zip_unzip_forecasts.R

View check run for this annotation

Codecov / codecov/patch

R/zip_unzip_forecasts.R#L50-L51

Added lines #L50 - L51 were not covered by tests
forecast_date == forecast_day,
select = c("forecast_id", "forecast_date")
)

# For each forecast_id get 3 files
All_ids <- newdata$"forecast_id"
for (id in All_ids) {
csv_file_path <- paste0(proj_path, "/forecast_id_", id, csv_file)
if (file.exists(csv_file_path)) {
id_date_files <- c(id_date_files, csv_file_path)
}
yaml_file_path <- paste0(proj_path, "/forecast_id_", id, yaml_file)
yaml_file_path <-
paste0(proj_path, "/forecast_id_", id, yaml_file)

Check warning on line 64 in R/zip_unzip_forecasts.R

View check run for this annotation

Codecov / codecov/patch

R/zip_unzip_forecasts.R#L63-L64

Added lines #L63 - L64 were not covered by tests
if (file.exists(yaml_file_path)) {
id_date_files <- c(id_date_files, yaml_file_path)
}
json_file_path <- paste0(proj_path, "/forecast_id_", id, json_file)
json_file_path <-
paste0(proj_path, "/forecast_id_", id, json_file)

Check warning on line 69 in R/zip_unzip_forecasts.R

View check run for this annotation

Codecov / codecov/patch

R/zip_unzip_forecasts.R#L68-L69

Added lines #L68 - L69 were not covered by tests
if (file.exists(json_file_path)) {
id_date_files <- c(id_date_files, json_file_path)
}
}

if (length(id_date_files)) {
# Zip forecast files for the current date
zip_files(zipfile, id_date_files)
}
}

# Zip forecasts_evaluations.csv
eval_file <- paste0(proj_path, "/forecasts_evaluations.csv")
zipfile <- paste0(proj_path, "/forecasts_evaluations.zip")
zip_files(zipfile, eval_file)
}

if (type == "unzip") {
messageq("Unzipping forecasts files")

# Unzip files based on unique_dates
for (forecast_day in unique_dates) {
zipfile <- paste0(proj_path, "/forecast_id_", forecast_day, ".zip")
Expand All @@ -91,7 +96,7 @@ zip_unzip <- function(type = NULL,
unlink(zipfile)
}
}

# Unzip forecasts_evaluations.csv
eval_zip <- paste0(proj_path, "/forecasts_evaluations.zip")
eval_zip <- normalizePath(eval_zip, mustWork = FALSE)
Expand Down
14 changes: 14 additions & 0 deletions man/post_process.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 0d42c4c

Please sign in to comment.