Merge pull request #223 from cmu-delphi/ndefries/add-missing-epidata-spec

Add missing field specs for endpoints; parse `release_date` as date

Author: nmdefries

R/endpoints.R

@@ -432,6 +432,64 @@ pub_covid_hosp_facility <- function(
       create_epidata_field_info(
         "total_patients_hosp_confirmed_influenza_and_covid_7d_avg",
         "float"
+      ),
+      create_epidata_field_info("geocoded_hospital_address", "text"),
+      create_epidata_field_info("hhs_ids", "text"),
+      create_epidata_field_info("is_corrected", "bool"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_7_day_coverage",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_7_day_coverage",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_pediatric_covid_confirmed_7_day_coverage",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_pediatric_covid_suspected_7_day_coverage",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_week_patients_covid_vaccinated_doses_all_7_day",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_week_patients_covid_vaccinated_doses_all_7_day_sum",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_week_patients_covid_vaccinated_doses_one_7_day",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_week_patients_covid_vaccinated_doses_one_7_day_sum",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_week_personnel_covid_vaccd_doses_administered_7_day",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_week_personnel_covid_vaccd_doses_administered_7_day_sum",
+        "int"
+      ),
+      create_epidata_field_info("total_personnel_covid_vaccinated_doses_all_7_day", "int"),
+      create_epidata_field_info(
+        "total_personnel_covid_vaccinated_doses_all_7_day_sum",
+        "int"
+      ),
+      create_epidata_field_info("total_personnel_covid_vaccinated_doses_none_7_day", "int"),
+      create_epidata_field_info(
+        "total_personnel_covid_vaccinated_doses_none_7_day_sum",
+        "int"
+      ),
+      create_epidata_field_info("total_personnel_covid_vaccinated_doses_one_7_day", "int"),
+      create_epidata_field_info(
+        "total_personnel_covid_vaccinated_doses_one_7_day_sum",
+        "int"
       )
     )
   ) %>% fetch(fetch_args = fetch_args)
@@ -618,7 +676,157 @@ pub_covid_hosp_state_timeseries <- function(
       create_epidata_field_info("percent_of_inpatients_with_covid", "float"),
       create_epidata_field_info("inpatient_bed_covid_utilization", "float"),
       create_epidata_field_info("adult_icu_bed_covid_utilization", "float"),
-      create_epidata_field_info("adult_icu_bed_utilization", "float")
+      create_epidata_field_info("adult_icu_bed_utilization", "float"),
+      create_epidata_field_info("geocoded_state", "text"),
+      create_epidata_field_info("deaths_covid", "int"),
+      create_epidata_field_info("deaths_covid_coverage", "int"),
+      create_epidata_field_info("icu_patients_confirmed_influenza", "int"),
+      create_epidata_field_info("icu_patients_confirmed_influenza_coverage", "int"),
+      create_epidata_field_info(
+        "on_hand_supply_therapeutic_a_casirivimab_imdevimab_courses",
+        "int"
+      ),
+      create_epidata_field_info("on_hand_supply_therapeutic_b_bamlanivimab_courses", "int"),
+      create_epidata_field_info(
+        "on_hand_supply_therapeutic_c_bamlanivimab_etesevimab_courses",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_confirmed_18_19", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_18_19_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_confirmed_20_29", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_20_29_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_confirmed_30_39", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_30_39_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_confirmed_40_49", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_40_49_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_confirmed_50_59", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_50_59_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_confirmed_60_69", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_60_69_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_confirmed_70_79", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_70_79_coverage",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_80plus",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_80plus_coverage",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_unknown",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_confirmed_unknown_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_suspected_18_19", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_18_19_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_suspected_20_29", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_20_29_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_suspected_30_39", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_30_39_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_suspected_40_49", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_40_49_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_suspected_50_59", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_50_59_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_suspected_60_69", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_60_69_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_adult_covid_suspected_70_79", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_70_79_coverage",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_80plus",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_80plus_coverage",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_unknown",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_day_admission_adult_covid_suspected_unknown_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_admission_influenza_confirmed", "int"),
+      create_epidata_field_info(
+        "previous_day_admission_influenza_confirmed_coverage",
+        "int"
+      ),
+      create_epidata_field_info("previous_day_deaths_covid_and_influenza", "int"),
+      create_epidata_field_info("previous_day_deaths_covid_and_influenza_coverage", "int"),
+      create_epidata_field_info("previous_day_deaths_influenza", "int"),
+      create_epidata_field_info("previous_day_deaths_influenza_coverage", "int"),
+      create_epidata_field_info(
+        "previous_week_therapeutic_a_casirivimab_imdevimab_courses_used",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_week_therapeutic_b_bamlanivimab_courses_used",
+        "int"
+      ),
+      create_epidata_field_info(
+        "previous_week_therapeutic_c_bamlanivimab_etesevimab_courses_used",
+        "int"
+      ),
+      create_epidata_field_info(
+        "total_patients_hospitalized_confirmed_influenza_covid",
+        "int"
+      ),
+      create_epidata_field_info(
+        "total_patients_hospitalized_confirmed_influenza_covid_coverage",
+        "int"
+      ),
+      create_epidata_field_info("total_patients_hospitalized_confirmed_influenza", "int"),
+      create_epidata_field_info(
+        "total_patients_hospitalized_confirmed_influenza_coverage",
+        "int"
+      )
     )
   ) %>% fetch(fetch_args = fetch_args)
 }
@@ -656,6 +864,11 @@ pub_covidcast_meta <- function(fetch_args = fetch_args_list()) {
         categories =
           c("week", "day")
       ),
+      create_epidata_field_info(
+        "geo_type",
+        "categorical",
+        categories = c("nation", "msa", "hrr", "hhs", "state", "county", "dma")
+      ),
       create_epidata_field_info("min_time", "date"),
       create_epidata_field_info("max_time", "date"),
       create_epidata_field_info("num_locations", "int"),
@@ -1031,7 +1244,7 @@ pub_flusurv <- function(locations, epiweeks, ..., issues = NULL, lag = NULL, fet
       lag = lag
     ),
     list(
-      create_epidata_field_info("release_date", "text"),
+      create_epidata_field_info("release_date", "date"),
       create_epidata_field_info("location", "text"),
       create_epidata_field_info("issue", "epiweek"),
       create_epidata_field_info("epiweek", "epiweek"),
@@ -1089,7 +1302,7 @@ pub_fluview_clinical <- function(regions, epiweeks, ..., issues = NULL, lag = NU
       lag = lag
     ),
     list(
-      create_epidata_field_info("release_date", "text"),
+      create_epidata_field_info("release_date", "date"),
       create_epidata_field_info("region", "text"),
       create_epidata_field_info("issue", "epiweek"),
       create_epidata_field_info("epiweek", "epiweek"),
@@ -1124,7 +1337,7 @@ pub_fluview_meta <- function(fetch_args = fetch_args_list()) {
     "fluview_meta/",
     list(),
     list(
-      create_epidata_field_info("latest_update", "text"),
+      create_epidata_field_info("latest_update", "date"),
       create_epidata_field_info("latest_issue", "date"),
       create_epidata_field_info("table_rows", "int")
     )
@@ -1198,13 +1411,14 @@ pub_fluview <- function(
       auth = auth
     ),
     list(
-      create_epidata_field_info("release_date", "text"),
+      create_epidata_field_info("release_date", "date"),
       create_epidata_field_info("region", "text"),
       create_epidata_field_info("issue", "epiweek"),
       create_epidata_field_info("epiweek", "epiweek"),
       create_epidata_field_info("lag", "int"),
       create_epidata_field_info("num_ili", "int"),
       create_epidata_field_info("num_patients", "int"),
+      create_epidata_field_info("num_providers", "int"),
       create_epidata_field_info("num_age_0", "int"),
       create_epidata_field_info("num_age_1", "int"),
       create_epidata_field_info("num_age_2", "int"),
@@ -1347,7 +1561,7 @@ pub_kcdc_ili <- function(regions, epiweeks, ..., issues = NULL, lag = NULL, fetc
       lag = lag
     ),
     list(
-      create_epidata_field_info("release_date", "text"),
+      create_epidata_field_info("release_date", "date"),
       create_epidata_field_info("region", "text"),
       create_epidata_field_info("issue", "epiweek"),
       create_epidata_field_info("epiweek", "epiweek"),
@@ -1480,7 +1694,7 @@ pub_nidss_flu <- function(regions, epiweeks, ..., issues = NULL, lag = NULL, fet
       lag = lag
     ),
     list(
-      create_epidata_field_info("release_date", "text"),
+      create_epidata_field_info("release_date", "date"),
       create_epidata_field_info("region", "text"),
       create_epidata_field_info("epiweek", "epiweek"),
       create_epidata_field_info("issue", "epiweek"),
@@ -1530,7 +1744,7 @@ pvt_norostat <- function(auth, locations, epiweeks, fetch_args = fetch_args_list
       epiweeks = epiweeks
     ),
     list(
-      create_epidata_field_info("release_date", "text"),
+      create_epidata_field_info("release_date", "date"),
       create_epidata_field_info("epiweek", "epiweek"),
       create_epidata_field_info("value", "int")
     )
@@ -1611,7 +1825,7 @@ pub_paho_dengue <- function(regions, epiweeks, ..., issues = NULL, lag = NULL, f
       lag = lag
     ),
     list(
-      create_epidata_field_info("release_date", "text"),
+      create_epidata_field_info("release_date", "date"),
       create_epidata_field_info("region", "text"),
       create_epidata_field_info("serotype", "text"),
       create_epidata_field_info("epiweek", "epiweek"),

R/model.R

@@ -174,7 +174,7 @@ parse_data_frame <- function(epidata_call, df, disable_date_parsing = FALSE) {
 
 #' @keywords internal
 parse_api_date <- function(value) {
-  as.Date(as.character(value), format = "%Y%m%d")
+  as.Date(as.character(value), tryFormats = c("%Y%m%d", "%Y-%m-%d"))
 }
 
 #' parse_api_week converts an integer to a date

tests/testthat/test-endpoints.R

@@ -37,14 +37,14 @@ test_that("basic_epidata_call", {
     fetch_args = fetch_args_list(dry_run = TRUE)
   ) %>% request_url())
   expect_no_error(pub_dengue_nowcast(
-    locations = "?",
+    locations = "ca",
     epiweeks = epirange(201501, 202001),
     fetch_args = fetch_args_list(dry_run = TRUE)
   ) %>% request_url())
   expect_no_error(pvt_dengue_sensors(
     auth = "yourkey",
-    names = "?",
-    locations = "?",
+    names = "ght",
+    locations = "ag",
     epiweeks = epirange(201501, 202001),
     fetch_args = fetch_args_list(dry_run = TRUE)
   ) %>% request_url())
@@ -80,11 +80,11 @@ test_that("basic_epidata_call", {
     auth = "yourkey",
     locations = "ca",
     epiweeks = epirange(201201, 202001),
-    query = "?",
+    query = "how to get over the flu",
     fetch_args = fetch_args_list(dry_run = TRUE)
   ) %>% request_url())
   expect_no_error(pub_kcdc_ili(
-    regions = "?",
+    regions = "ROK",
     epiweeks = epirange(201201, 202001),
     fetch_args = fetch_args_list(dry_run = TRUE)
   ) %>% request_url())
@@ -213,7 +213,7 @@ test_that("endpoints fail when given args via dots", {
   )
   expect_error(
     pub_kcdc_ili(
-      regions = "?",
+      regions = "ROK",
       date_range = epirange(201201, 202001)
     ),
     regexp = dots_error

tests/testthat/test-model.R

@@ -57,11 +57,12 @@ test_that("null parsing", {
   # see generate_test_data.R
   mock_df <- as.data.frame(readr::read_rds(testthat::test_path("data/flusurv-epiweeks.rds")))
   metadata <- epidata_call$meta
-  mock_df[[metadata[[1]]$name]][1] <- list(NULL)
+  mock_df[[metadata[[1]]$name]][1] <- NA
   mock_df[[metadata[[2]]$name]] <- c(TRUE)
   epidata_call$meta[[2]]$type <- "bool"
-  res <- parse_data_frame(epidata_call, mock_df) %>% as_tibble()
+  expect_no_error(res <- parse_data_frame(epidata_call, mock_df) %>% as_tibble())
   expect_true(res$location)
+  expect_identical(res$release_date, as.Date(NA))
 
   # if the call has no metadata, return the whole frame as is
   epidata_call$meta <- NULL
@@ -98,3 +99,17 @@ test_that("parse_data_frame warns when df contains fields not listed in meta", {
   mock_df$rate_age_0 <- NULL
   expect_no_warning(parse_data_frame(epidata_call, mock_df))
 })
+
+test_that("parse_api_date accepts str and int input", {
+  expect_identical(parse_api_date("20200101"), as.Date("2020-01-01"))
+  expect_identical(parse_api_date(20200101), as.Date("2020-01-01"))
+})
+
+test_that("parse_api_date accepts YYYYMMDD and YYYY-MM-DD", {
+  expect_identical(parse_api_date(20200101), as.Date("2020-01-01"))
+  expect_identical(parse_api_date("2020-01-01"), as.Date("2020-01-01"))
+})
+
+test_that("parse_api_date handles missing values appropriately", {
+  expect_identical(parse_api_date(NA), as.Date(NA))
+})