Skip to content

Commit

Permalink
0.0.5 - granular file type control in nflverse_save
Browse files Browse the repository at this point in the history
  • Loading branch information
tanho63 committed May 29, 2022
1 parent 9310dc7 commit 55a5b33
Show file tree
Hide file tree
Showing 6 changed files with 46 additions and 17 deletions.
4 changes: 2 additions & 2 deletions DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Package: nflversedata
Title: nflverse data storage functions
Version: 0.0.4
Version: 0.0.5
Authors@R:
person("Tan", "Ho", , "[email protected]", role = c("aut", "cre"),
comment = c(ORCID = "0000-0001-8388-5155"))
Expand Down Expand Up @@ -28,4 +28,4 @@ Remotes:
Encoding: UTF-8
LazyData: true
Roxygen: list(markdown = TRUE)
RoxygenNote: 7.1.2
RoxygenNote: 7.2.0
1 change: 1 addition & 0 deletions NAMESPACE
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Generated by roxygen2: do not edit by hand

export(nflverse_archive)
export(nflverse_save)
export(nflverse_upload)
4 changes: 4 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# nflversedata 0.0.5

* Added file_types argument which takes a vector of file types and defaults to `c("csv","rds","qs","parquet")`

# nflversedata 0.0.4

* Added `nflverse_archive()` which automates archiving the rds version of every nflverse-data release asset and pushes it to https://github.com/nflverse/nflverse-data-archives
Expand Down
36 changes: 22 additions & 14 deletions R/upload.R
Original file line number Diff line number Diff line change
Expand Up @@ -40,45 +40,53 @@ update_release_timestamp <- function(tag){
#' @param nflverse_type metadata: name/information to add to data
#' @param release_tag name of release to upload to
#' @param .token a GitHub token, defaults to gh::gh_token()
#' @param file_types one or more of c("rds","csv","parquet","qs","csv.gz")
#'
#' @export
nflverse_save <- function(data_frame,
file_name,
nflverse_type,
release_tag,
.token = gh::gh_token(),
include_gz = FALSE
){
file_types = c("rds","csv","parquet","qs")
){

stopifnot(
is.data.frame(data_frame),
is.character(file_name),
is.character(nflverse_type),
is.character(release_tag),
is.character(.token),
is.character(file_types),
length(file_name) == 1,
length(nflverse_type) == 1,
length(release_tag) == 1,
length(.token) == 1
length(.token) == 1,
length(file_types) >= 1
)

attr(data_frame,"nflverse_type") <- nflverse_type
attr(data_frame,"nflverse_timestamp") <- Sys.time()

temp_dir <- tempdir(check = TRUE)
ft <- rlang::arg_match(file_types,
values = c("rds","csv","csv.gz","parquet","qs"),
multiple = TRUE)

saveRDS(data_frame,file.path(temp_dir,paste0(file_name,".rds")))
data.table::fwrite(data_frame, file.path(temp_dir,paste0(file_name,".csv")))
if(include_gz) data.table::fwrite(data_frame, file.path(temp_dir,paste0(file_name,".csv.gz")))
arrow::write_parquet(data_frame, file.path(temp_dir, paste0(file_name,".parquet")))
qs::qsave(data_frame,
file.path(temp_dir,paste0(file_name,".qs")),
preset = "custom",
algorithm = "zstd_stream",
compress_level = 22,
shuffle_control = 15)
if("rds" %in% ft) saveRDS(data_frame,file.path(temp_dir,paste0(file_name,".rds")))
if("csv" %in% ft) data.table::fwrite(data_frame, file.path(temp_dir,paste0(file_name,".csv")))
if("csv.gz" %in% ft) data.table::fwrite(data_frame, file.path(temp_dir,paste0(file_name,".csv.gz")))
if("parquet" %in% ft) arrow::write_parquet(data_frame, file.path(temp_dir, paste0(file_name,".parquet")))
if("qs" %in% ft){
qs::qsave(data_frame,
file.path(temp_dir,paste0(file_name,".qs")),
preset = "custom",
algorithm = "zstd_stream",
compress_level = 22,
shuffle_control = 15)
}

.filetypes <- if(include_gz) c(".rds",".csv",".csv.gz",".parquet",".qs") else c(".rds",".csv",".parquet",".qs")
.filetypes <- paste0(".",ft)

.file_names <- file.path(temp_dir, paste0(file_name,.filetypes))

Expand Down
14 changes: 14 additions & 0 deletions man/nflverse_archive.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion man/nflverse_save.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 55a5b33

Please sign in to comment.