From d021db7cb1787b13fff53d36f985ddc85d1b4ba1 Mon Sep 17 00:00:00 2001 From: Noam Ross Date: Tue, 28 Nov 2023 11:35:07 -0500 Subject: [PATCH 1/2] Fix problems due to differing prefixes in S3 stores --- R/tar_read_version.R | 52 +++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/R/tar_read_version.R b/R/tar_read_version.R index c554580..1be5f97 100644 --- a/R/tar_read_version.R +++ b/R/tar_read_version.R @@ -54,49 +54,61 @@ tar_read_raw_version <- function(name, ref = "HEAD", repo = ".", store = NULL) { } target <- switch(record$repository, - local = read_target_aws(record, path_store = path_store), - aws = read_target_aws(record, path_store = path_store), - abort("Unknown targets repository type: ", record$repository) + local = read_target_aws(record), + aws = read_target_aws(record), + abort("Unknown targets repository type: ", record$repository) ) target } -read_target_aws <- function(record, path_store) { +read_target_aws <- function(record) { aws_loc <- aws_loc_from_meta_path(record$path[[1]]) local_target_path <- get_file_version( path = aws_loc$key, ref = aws_loc$version, repo = paste0("s3://", aws_loc$bucket), endpoint = aws_loc$endpoint, region = aws_loc$region ) - record_local <- record - record_local$path <- list(local_target_path) - record_local$repository <- "local" - targets::tar_read_raw(record_local$name, - meta = record_local, - store = path_dir(path_dir(local_target_path)) - ) + if (record$format == "file") { + return(local_target_path) + } else { + record_local <- record + record_local$path <- NA + record_local$repository <- "local" + temp_store <- path_dir(dir_create(path(file_temp("_targets"), "objects"))) + link_create(local_target_path, path(temp_store, "objects", path_file(local_target_path))) + on.exit(dir_delete(temp_store)) + return(targets::tar_read_raw(record_local$name, + meta = record_local, + store = temp_store)) + } } -read_target_local <- function(record, path_store) { +read_target_local <- function(record) { # For local targets local_target_path <- get_file_version( path = record$path[[1]], ref = record$version, repo = record$repository ) - record_local <- record - record_local$path <- list(local_target_path) - - targets::tar_read_raw(record_local$name, - meta = record_local, - store = path_store - ) + if (record$format == "file") { + return(local_target_path) + } else { + record_local <- record + record_local$path <- NA + record_local$repository <- "local" + temp_store <- path_dir(dir_create(path(file_temp("_targets"), "objects"))) + link_create(local_target_path, path(temp_store, "objects", path_file(local_target_path))) + on.exit(dir_delete(temp_store)) + return(targets::tar_read_raw(record_local$name, + meta = record_local, + store = temp_store)) + } } aws_loc_from_meta_path <- function(path) { splits <- strsplit(path, "=") aws_loc <- structure(lapply(splits, function(x) x[[2]]), - .Names = vapply(splits, function(x) x[[1]], character(1)) + .Names = vapply(splits, function(x) x[[1]], character(1)) ) if (!is.null(aws_loc$endpoint)) { aws_loc$endpoint <- rawToChar(openssl::base64_decode(aws_loc$endpoint)) From e86e52b6a70c8591bfafd3129d394244a589d557 Mon Sep 17 00:00:00 2001 From: Noam Ross Date: Wed, 29 Nov 2023 10:43:34 -0500 Subject: [PATCH 2/2] Change the maximum cache size default --- R/cache.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/cache.R b/R/cache.R index 046162f..37bddac 100644 --- a/R/cache.R +++ b/R/cache.R @@ -32,7 +32,7 @@ relic_cache_delete <- function() { #' `options("relic.cache.max.age")`, which take numeric time in days or a #' string with units, e.g., "1 day" or "2 weeks". #' @param max_size The maximum size of the cache, as a string that can be parsed -#' by [fs::fs_bytes()]. Defaults to "20 MB". Can be set with the environment +#' by [fs::fs_bytes()]. Defaults to "100 GB". Can be set with the environment #' variable `RELIC_CACHE_MAX_SIZE` or `options("relic.cache.max.size")`. #' Cached files will be deleted from oldest to youngest until the cache size #' is under this limit. @@ -84,7 +84,7 @@ relic_cache_max_size <- function() { fs_bytes(Sys.getenv( "RELIC_CACHE_MAX_SIZE", getOption("relic.cache.max.size", - default = "20 MB" + default = "100 GB" ) )) }