Merge branch 'PecanProject:develop' into Meettest

allgandalf · Nov 13, 2023 · 5ac3dd5 · 5ac3dd5
2 parents 25e864d + 91531ae
commit 5ac3dd5
Show file tree

Hide file tree

Showing 21 changed files with 206 additions and 215 deletions.
diff --git a/book_source/_bookdown.yml b/book_source/_bookdown.yml
@@ -1,2 +1,2 @@
 rmd_subdir: true
-edit: https://github.com/tonygardella/pecan/edit/release/vtonydoc/book_source/%s
+edit: https://github.com/PecanProject/pecan/edit/develop/book_source/%s
diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R
@@ -9,6 +9,7 @@ Sys.setenv(RLIB = rlib)
 
 # install remotes first in case packages are references in dependencies
 remotes::install_github(c(
+'adokter/suntools',
 'araiho/linkages_package',
 'chuhousen/amerifluxr',
 'ebimodeling/[email protected]',
@@ -76,7 +77,6 @@ wanted <- c(
 'magic',
 'magrittr',
 'maps',
-'maptools',
 'markdown',
 'MASS',
 'Matrix',
@@ -113,7 +113,6 @@ wanted <- c(
 'reshape',
 'reshape2',
 'reticulate',
-'rgdal',
 'rjags',
 'rjson',
 'rlang',
@@ -130,6 +129,7 @@ wanted <- c(
 'stats',
 'stringi',
 'stringr',
+'suntools',
 'swfscMisc',
 'terra',
 'testthat',

diff --git a/modules/data.atmosphere/DESCRIPTION b/modules/data.atmosphere/DESCRIPTION
@@ -36,7 +36,6 @@ Imports:
     jsonlite,
     lubridate (>= 1.6.0),
     magrittr,
-    maptools,
     MASS,
     mgcv,
     ncdf4 (>= 1.15),
@@ -49,10 +48,12 @@ Imports:
     raster,
     REddyProc,
     reshape2,
-    rgdal,
     rlang (>= 0.2.0),
+    sf,
     sp,
     stringr (>= 1.1.0),
+    suntools,
+    terra,
     testthat (>= 2.0.0),
     tibble,
     tidyr,
@@ -69,6 +70,7 @@ Suggests:
     progress,
     reticulate
 Remotes:
+    github::adokter/suntools,
     github::chuhousen/amerifluxr,
     github::ropensci/geonames,
     github::ropensci/nneo

diff --git a/modules/data.atmosphere/NAMESPACE b/modules/data.atmosphere/NAMESPACE
@@ -113,5 +113,5 @@ export(write_noaa_gefs_netcdf)
 import(dplyr)
 import(tidyselect)
 importFrom(magrittr,"%>%")
-importFrom(rgdal,checkCRSArgs)
 importFrom(rlang,.data)
+importFrom(sf,st_crs)
diff --git a/modules/data.atmosphere/R/GEFS_helper_functions.R b/modules/data.atmosphere/R/GEFS_helper_functions.R
@@ -243,25 +243,30 @@ process_gridded_noaa_download <- function(lat_list,
 
     for(hr in 1:length(curr_hours)){
       file_name <- paste0(base_filename2, curr_hours[hr])
+      grib_file_name <- paste0(working_directory,"/", file_name,".grib")
 
-      if(file.exists(paste0(working_directory,"/", file_name,".grib"))){
-        grib <- rgdal::readGDAL(paste0(working_directory,"/", file_name,".grib"), silent = TRUE)
-        lat_lon <- sp::coordinates(grib)
+      if(file.exists(grib_file_name)){
+        grib_data <- terra::rast(grib_file_name)
+
+        ## Convert to data frame
+        grib_data_df <- terra::as.data.frame(grib_data, xy=TRUE)
+        lat_lon <- grib_data_df[, c("x", "y")]
+
         for(s in 1:length(site_id)){
 
           index <- which(lat_lon[,2] == lats[s] & lat_lon[,1] == lons[s])
 
-          pressfc[s, hr]  <- grib$band1[index]
-          tmp2m[s, hr] <- grib$band2[index]
-          rh2m[s, hr]  <- grib$band3[index]
-          ugrd10m[s, hr]  <- grib$band4[index]
-          vgrd10m[s, hr]  <- grib$band5[index]
+          pressfc[s, hr]  <- grib_data_df$`SFC=Ground or water surface; Pressure [Pa]`[index]
+          tmp2m[s, hr]    <- grib_data_df$`2[m] HTGL=Specified height level above ground; Temperature [C]`[index]
+          rh2m[s, hr]     <- grib_data_df$`2[m] HTGL=Specified height level above ground; Relative humidity [%]`[index]
+          ugrd10m[s, hr]  <- grib_data_df$`10[m] HTGL=Specified height level above ground; u-component of wind [m/s]`[index]
+          vgrd10m[s, hr]  <- grib_data_df$`10[m] HTGL=Specified height level above ground; v-component of wind [m/s]`[index]
 
           if(curr_hours[hr] != "000"){
-            apcpsfc[s, hr]  <- grib$band6[index]
-            tcdcclm[s, hr]  <-  grib$band7[index]
-            dswrfsfc[s, hr]  <- grib$band8[index]
-            dlwrfsfc[s, hr]  <- grib$band9[index]
+            apcpsfc[s, hr]  <- grib_data_df$`SFC=Ground or water surface; 03 hr Total precipitation [kg/(m^2)]`[index]
+            tcdcclm[s, hr]  <- grib_data_df$`RESERVED(10) (Reserved); Total cloud cover [%]`[index]
+            dswrfsfc[s, hr] <- grib_data_df$`SFC=Ground or water surface; Downward Short-Wave Rad. Flux [W/(m^2)]`[index]
+            dlwrfsfc[s, hr] <- grib_data_df$`SFC=Ground or water surface; Downward Long-Wave Rad. Flux [W/(m^2)]`[index]
           }
         }
       }

diff --git a/modules/data.atmosphere/R/download.NARR_site.R b/modules/data.atmosphere/R/download.NARR_site.R
@@ -465,9 +465,9 @@ latlon2narr <- function(nc, lat.in, lon.in) {
 #' @inheritParams get_NARR_thredds
 #' @return `sp::SpatialPoints` object containing transformed x and y
 #' coordinates, in km, which should match NARR coordinates
-#' @importFrom rgdal checkCRSArgs
+#' @importFrom sf st_crs
   # ^not used directly here, but needed by sp::CRS.
-  # sp lists rgdal in Suggests rather than Imports,
+  # sp lists sf in Suggests rather than Imports,
   # so importing it here to ensure it's available at run time
 #' @author Alexey Shiklomanov
 #' @export

diff --git a/modules/data.atmosphere/R/met_temporal_downscale.Gaussian_ensemble.R b/modules/data.atmosphere/R/met_temporal_downscale.Gaussian_ensemble.R
@@ -216,11 +216,11 @@ met_temporal_downscale.Gaussian_ensemble <- function(in.path, in.prefix, outfold
       day <- as.POSIXct(sprintf("%s 12:00:00", date), tz = tz)
       sequence <- seq(from = day, length.out = span, by = "days")
 
-      sunrise <- maptools::sunriset(lon.lat, sequence, direction = "sunrise", 
+      sunrise <- suntools::sunriset(lon.lat, sequence, direction = "sunrise", 
                                     POSIXct.out = TRUE)
-      sunset <- maptools::sunriset(lon.lat, sequence, direction = "sunset", 
+      sunset <- suntools::sunriset(lon.lat, sequence, direction = "sunset", 
                                    POSIXct.out = TRUE)
-      solar_noon <- maptools::solarnoon(lon.lat, sequence, POSIXct.out = TRUE)
+      solar_noon <- suntools::solarnoon(lon.lat, sequence, POSIXct.out = TRUE)
 
       data.frame(date = as.Date(sunrise$time), sunrise = as.numeric(format(sunrise$time, "%H%M")), 
                  solarnoon = as.numeric(format(solar_noon$time, "%H%M")), 

diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R b/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R
@@ -1,4 +1,5 @@
 library(testthat)
+library(PEcAn.DB)
 
 test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, lon.in) {
   # putting logger to debug mode
@@ -7,13 +8,13 @@ test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, l
   PEcAn.logger::logger.setLevel("DEBUG")
 
   # mocking functions
-  mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame())
-  mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1))
+  mockery::stub(convert_input, 'dbfile.input.check', data.frame())
+  mockery::stub(convert_input, 'db.query', data.frame(id = 1))
 
   withr::with_dir(tempdir(), {
     tmpdir <- getwd()
     # calling download function
-    res <- PEcAn.DB::convert_input(
+    res <- convert_input(
       input.id = NA,
       outfolder = tmpdir,
       formatname = NULL,
@@ -31,23 +32,23 @@ test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, l
       lon.in = lon.in,
       sitename = sitename
     )
-  })
-
-  # checking if the file is downloaded
-  test_that("Downloaded files are present at the desired location", {
-    expect_true(file.exists(paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")))
-  })
 
-  test_that("Downloaded data files have the right format", {
-    firstline <- system(paste0("head -4 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
-    lastline <- system(paste0("tail -1 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
-
-    # checking if first line of CSV has the sitename
-    expect_true(grepl(sitename, firstline[1]))
-
-    # fourth and last row checked to contain non-alphabetical data since these are used to verify start and end dates
-    expect_false(grepl("[A-Za-z]", firstline[4]))
-    expect_false(grepl("[A-Za-z]", lastline[1]))
+    # checking if the file is downloaded
+    test_that("Downloaded files are present at the desired location", {
+      expect_true(file.exists(paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")))
+    })
+
+    test_that("Downloaded data files have the right format", {
+      firstline <- system(paste0("head -4 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
+      lastline <- system(paste0("tail -1 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
+
+      # checking if first line of CSV has the sitename
+      expect_true(grepl(sitename, firstline[1]))
+
+      # fourth and last row checked to contain non-alphabetical data since these are used to verify start and end dates
+      expect_false(grepl("[A-Za-z]", firstline[4]))
+      expect_false(grepl("[A-Za-z]", lastline[1]))
+    })
   })
 }
 

diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R b/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R
@@ -1,5 +1,6 @@
 library(testthat)
 library(ncdf4)
+library(PEcAn.DB)
 
 test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method, maxErrors, sleep) {
   # putting logger to debug mode
@@ -8,12 +9,12 @@ test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method,
   PEcAn.logger::logger.setLevel("DEBUG")
 
   # mocking functions
-  mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame())
-  mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1))
+  mockery::stub(convert_input, 'dbfile.input.check', data.frame())
+  mockery::stub(convert_input, 'db.query', data.frame(id = 1))
 
   withr::with_dir(tempdir(), {
     tmpdir <- getwd()
-    PEcAn.DB::convert_input(
+    convert_input(
       input.id = NA,
       outfolder = tmpdir,
       formatname = NULL,

diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R b/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R
@@ -1,5 +1,6 @@
 library(testthat)
 library(ncdf4)
+library(PEcAn.DB)
 
 test_download_ERA5 <- function(start_date, end_date, lat.in, lon.in, product_types, reticulate_python) {
   # putting logger to debug mode
@@ -9,16 +10,16 @@ test_download_ERA5 <- function(start_date, end_date, lat.in, lon.in, product_typ
 
 
   # mocking functions
-  mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame())
-  mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1))
+  mockery::stub(convert_input, 'dbfile.input.check', data.frame())
+  mockery::stub(convert_input, 'db.query', data.frame(id = 1))
 
   # additional mocks needed since download.ERA5 does not return data as other download functions
-  mockery::stub(PEcAn.DB::convert_input, 'length', 2)
-  mockery::stub(PEcAn.DB::convert_input, 'purrr::map_dfr', data.frame(missing = c(FALSE), empty = c(FALSE)))
+  mockery::stub(convert_input, 'length', 2)
+  mockery::stub(convert_input, 'purrr::map_dfr', data.frame(missing = c(FALSE), empty = c(FALSE)))
 
   withr::with_dir(tempdir(), {
     tmpdir <- getwd()
-    PEcAn.DB::convert_input(
+    convert_input(
       input.id = NA,
       outfolder = tmpdir,
       formatname = NULL,
@@ -37,51 +38,51 @@ test_download_ERA5 <- function(start_date, end_date, lat.in, lon.in, product_typ
       product_types = product_types,
       reticulate_python = reticulate_python
     )
-  })
-
-  test_that("All the required files are downloaded and stored at desired location", { 
-    expect_true(file.exists(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.2m_temperature.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.surface_pressure.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.total_precipitation.nc")))
-  })
+
+    test_that("All the required files are downloaded and stored at desired location", { 
+      expect_true(file.exists(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.2m_temperature.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.surface_pressure.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.total_precipitation.nc")))
+    })
 
-  test_that("All ERA5 data files have the correct variable units", {
-    nc <- nc_open(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc"))
-    expect_equal(nc$var$d2m$units, "K")
-    nc_close(nc)
+    test_that("All ERA5 data files have the correct variable units", {
+      nc <- nc_open(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc"))
+      expect_equal(nc$var$d2m$units, "K")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.2m_temperature.nc"))
-    expect_equal(nc$var$t2m$units, "K")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.2m_temperature.nc"))
+      expect_equal(nc$var$t2m$units, "K")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc"))
-    expect_equal(nc$var$u10$units, "m s**-1")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc"))
+      expect_equal(nc$var$u10$units, "m s**-1")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc"))
-    expect_equal(nc$var$v10$units, "m s**-1")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc"))
+      expect_equal(nc$var$v10$units, "m s**-1")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.surface_pressure.nc"))
-    expect_equal(nc$var$sp$units, "Pa")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.surface_pressure.nc"))
+      expect_equal(nc$var$sp$units, "Pa")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc"))
-    expect_equal(nc$var$ssrd$units, "J m**-2")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc"))
+      expect_equal(nc$var$ssrd$units, "J m**-2")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc"))
-    expect_equal(nc$var$strd$units, "J m**-2")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc"))
+      expect_equal(nc$var$strd$units, "J m**-2")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.total_precipitation.nc"))
-    expect_equal(nc$var$tp$units, "m")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.total_precipitation.nc"))
+      expect_equal(nc$var$tp$units, "m")
+      nc_close(nc)
+    })
   })
 }
 

diff --git a/modules/data.atmosphere/tests/Rcheck_reference.log b/modules/data.atmosphere/tests/Rcheck_reference.log
@@ -76,13 +76,13 @@ Found the following (possibly) invalid URLs:
 The Date field is over a month old.
 * checking package namespace information ... OK
 * checking package dependencies ... WARNING
-Imports includes 39 non-default packages.
+Imports includes 40 non-default packages.
 Importing from so many packages makes the package vulnerable to any of
 them becoming unavailable.  Move as many as possible to Suggests and
 use conditionally.
 
 * checking package dependencies ... NOTE
-Imports includes 39 non-default packages.
+Imports includes 40 non-default packages.
 Importing from so many packages makes the package vulnerable to any of
 them becoming unavailable.  Move as many as possible to Suggests and
 use conditionally.

diff --git a/modules/data.land/DESCRIPTION b/modules/data.land/DESCRIPTION
@@ -61,7 +61,6 @@ Suggests:
     PEcAn.settings,
     redland,
     raster,
-    rgdal,
     RPostgreSQL,
     testthat (>= 1.0.2)
 License: BSD_3_clause + file LICENSE

diff --git a/modules/data.land/R/gis.functions.R b/modules/data.land/R/gis.functions.R
@@ -68,14 +68,13 @@ shp2kml <- function(dir, ext, kmz = FALSE, proj4 = NULL, color = NULL, NameField
     # Read in shapefile(s) & get coordinates/projection info shp.file <-
     # readShapeSpatial(file.path(dir,i),verbose=TRUE) coordinates(test) <- ~X+Y
 
-    layers <- rgdal::ogrListLayers(file.path(dir, i))
-    info   <- rgdal::ogrInfo(file.path(dir, i), layers)
+    layers <- sf::st_layers(file.path(dir, i))
     # shp.file <- readOGR(file.path(dir,i),layer=layers) # no need to read in file
 
     # Display vector info to the console
     print("")
-    print(paste0("Input layers: ", layers))
-    print(paste0("Input projection info: ", info$p4s))
+    print(paste0("Input layers: ", layers$name))
+    print(paste0("Input projection info: ", layers$crs[[1]]$input))
     print("")
 
     # Write out kml/kmz using plotKML package if (is.null(color)){ color <- 'grey70' }
@@ -133,6 +132,7 @@ get.attributes <- function(file, coords) {
 
   #library(fields)
   #require(rgdal)
+  # note that OGR support is now provided by the sf and terra packages among others 
 
   # print('NOT IMPLEMENTED YET') subset_layer(file,coords)
 } # get.attributes