correction of the extract_data and get_eurostat_data function

eurostat · Sep 14, 2023 · 0d835b5 · 0d835b5
1 parent 7ea2b41
commit 0d835b5
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 9 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,8 +1,8 @@
 Package: restatapi
 Type: Package
 Title: Search and Retrieve Data from Eurostat Database
-Date: 2023-09-14
-Version: 0.21.3
+Date: 2023-09-15
+Version: 0.22.0
 Encoding: UTF-8
 Authors@R: c(person("Mátyás", "Mészáros", email = "[email protected]", role = c("aut", "cre")),
             person("Sebastian", "Weinand", role = "ctb"))

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,8 @@
+# restatapi 0.22.0
+
+- correction of the `extract_data()` function to handle observations when the value is not numeric
+- correction of the `get_eurostat_data()` function when the option `force_local_filter=TRUE`
+
 # restatapi 0.21.3
 
 - correction of tests

diff --git a/R/extract_data.R b/R/extract_data.R
@@ -83,8 +83,8 @@ extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE,
                                                 }
                                                 return(as.data.frame(dr,stringsAsFactors=FALSE))
       }), fill=TRUE)
-    # if (!grepl("\\d:\\d",df$obsValue)) df$obsValue<-as.numeric(df$obsValue)
-    df$obsValue<-as.numeric(df$obsValue)
+    if (any(!grepl("\\d:\\d",df$obsValue))) df$obsValue<-as.numeric(df$obsValue)
+    # df$obsValue<-as.numeric(df$obsValue)
     df$obsValue[df$obsValue=="NaN"]<-NA
     if (keep_flags){
       out<-data.frame(bd,df,stringsAsFactors=stringsAsFactors)  

diff --git a/R/get_eurostat_data.R b/R/get_eurostat_data.R
@@ -351,19 +351,21 @@ get_eurostat_data <- function(id,
         message("None of the filter could be applied. The whole dataset will be retrieved through bulk download.")
         restat<-restatapi::get_eurostat_bulk(id,cache,update_cache,cache_dir,compress_file,stringsAsFactors,select_freq,keep_flags,cflags,check_toc,verbose=verbose)
         if (!is.null(restat) & (verbose)) {message("get_eurostat_data - bulk restat - nrow:",nrow(restat),";ncol:",ncol(restat),";colnames:",paste(colnames(restat),collapse="/"));message("cflags:",cflags)}
-      } else  if (force_local_filter) #there is valid filter but want to filter localy, not using the API and filter url => raw download and filtering
+      } else  if (force_local_filter) #there is valid filter but want to filter locally, not using the API and filter url => raw download and filtering
       { 
-        message("Forcing to apply filter locally. The whole dataset is downloaded through the raw download and the filters are applied locally.")
+        message("Forcing to apply filter locally. The whole dataset will be downloaded through the raw download and then the filters are applied locally.")
         restat_raw<-restatapi::get_eurostat_raw(id,"txt",cache,update_cache,cache_dir,compress_file,stringsAsFactors,keep_flags,check_toc,melt=TRUE,verbose)
         if (!is.null(restat_raw) & (verbose)) {message("get_eurostat_data - raw restat - nrow:",nrow(restat_raw),";ncol:",ncol(restat_raw),";colnames:",paste(colnames(restat_raw),collapse="/"))}
         if (verbose) {message("get_eurostat_data - filter table:");print(ft)}
-        if (!is.null(dft)){
+        if (!is.null(ft)){
           if (nrow(ft)>0){restat_raw<-restatapi::filter_raw_data(restat_raw,ft)[]}
         }
+        if (!is.null(restat_raw) & (verbose)) {message("get_eurostat_data - local filtered restat_raw after ft- nrow:",nrow(restat_raw),";ncol:",ncol(restat_raw),";colnames:",paste(colnames(restat_raw),collapse="/"))}
         if (verbose) {message("get_eurostat_data - date filter table:");print(dft)}
         if (!is.null(dft)){
           if (nrow(dft)>0){restat_raw<-restatapi::filter_raw_data(restat_raw,dft,TRUE)[]}
         }
+        if (!is.null(restat_raw) & (verbose)) {message("get_eurostat_data - local filtered restat_raw after dft - nrow:",nrow(restat_raw),";ncol:",ncol(restat_raw),";colnames:",paste(colnames(restat_raw),collapse="/"))}
         cr<-FALSE
         restat<-restat_raw[]
         if (!is.null(restat) & (verbose)) {message("get_eurostat_data - local filtered restat - nrow:",nrow(restat),";ncol:",ncol(restat),";colnames:",paste(colnames(restat),collapse="/"))}
@@ -660,9 +662,10 @@ get_eurostat_data <- function(id,
             restat$values<-suppressWarnings(as.numeric(levels(restat$values))[restat$values])        
           }
         }
-        if (is.character(restat$values)){
+        if (any(is.character(restat$values))){
           if (!any(grepl('\\d+\\:\\d+',restat$values))){
-            restat$values<-as.numeric(restat$values)
+            restat$values[grepl('^\\:$',restat$values)]<-NA
+            restat$values<-suppressWarnings(as.numeric(restat$values))
           }
         }
         restat<-unique(restat)[]