update of test and documentation

eurostat · Sep 18, 2023 · 179b042 · 179b042
1 parent 4ed7f0b
commit 179b042
Show file tree

Hide file tree

Showing 13 changed files with 423 additions and 490 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,8 +1,8 @@
 Package: restatapi
 Type: Package
 Title: Search and Retrieve Data from Eurostat Database
-Date: 2023-09-15
-Version: 0.22.0
+Date: 2023-09-19
+Version: 0.22.1
 Encoding: UTF-8
 Authors@R: c(person("Mátyás", "Mészáros", email = "[email protected]", role = c("aut", "cre")),
             person("Sebastian", "Weinand", role = "ctb"))

diff --git a/NEWS.md b/NEWS.md
@@ -1,5 +1,10 @@
 # restatapi 0.22.0
 
+- refactoring tests for CRAN checks
+- updating debugging messages and documentation
+
+# restatapi 0.22.0
+
 - correction of the `extract_data()` function to handle observations when the value is not numeric
 - correction of the `get_eurostat_data()` function when the option `force_local_filter=TRUE`
 

diff --git a/R/get_compressed_sdmx.R b/R/get_compressed_sdmx.R
@@ -42,8 +42,12 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){
                    })
         } else {
           tryCatch({utils::download.file(url,temp,dmethod,quiet=TRUE)},
-                   error = function(e) {tbc<-FALSE},
-                   warning = function(w) {tbc<-FALSE})
+                   error = function(e) {
+                     message("There is an error by the download of the SDMX file. Run the same command with verbose=TRUE option to get more info on the issue.")
+                     tbc<-FALSE},
+                   warning = function(w) {
+                     message("There is a warning by the download of the SDMX file. Run the same command with verbose=TRUE option to get more info on the issue.")
+                     tbc<-FALSE})
         }
 
         # unzip file
@@ -64,8 +68,8 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){
                    })
         } else {
           tryCatch({xml_fajl<-utils::unzip(temp,paste0(fajl,".xml"),exdir=tmpdir)},
-                   error = function(e) {},
-                   warning = function(w) {})
+                   error = function(e) { message("There is an error by the unzipping of the downloaded SDMX file. Run the same command with verbose=TRUE option to get more info on the issue.")},
+                   warning = function(w) { message("There is an warning by the unzipping of the downloaded SDMX file. Run the same command with verbose=TRUE option to get more info on the issue.")})
         }
         if (!is.null(xml_fajl)){xml<-xml2::read_xml(xml_fajl)} else {xml<-NULL}
           unlink(temp)
@@ -81,8 +85,8 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){
                  })
         } else {
           tryCatch({xml<-xml2::read_xml(gzcon(url(url,open="rb")))},
-                 error = function(e) {},
-                 warning = function(w) {})
+                 error = function(e) { message("There is an error by the download of the SDMX file. Run the same command with verbose=TRUE option to get more info on the issue.")},
+                 warning = function(w) { message("There is an warning by the download of the SDMX file. Run the same command with verbose=TRUE option to get more info on the issue.")})
         }
         if (verbose) message("get_compressed_sdmx - xml NULL:",is.null(xml))  
         # if (!is.null(xml_fajl)){xml<-xml2::read_xml(gzcon(url(url,open="rb")))} else {xml<-NULL}

diff --git a/R/get_eurostat_bulk.R b/R/get_eurostat_bulk.R
@@ -35,7 +35,7 @@
 #' @param ... other parameter(s) to pass on the \code{\link{load_cfg}} function        
 #' @export
 #' 
-#' @details Data sets are downloaded from \href{https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing}{the Eurostat bulk download facility} 
+#' @details Data sets are downloaded from \href{https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API}{the Eurostat bulk download facility} 
 #' in TSV format as in this case smaller file has to be downloaded and processed. If there is more then one frequency then
 #'  the dataset is filtered for a unique time frequency.
 #' If no frequency is selected and there are multiple frequencies in the dataset, then the most common value is used used for frequency.

diff --git a/R/get_eurostat_codelist.R b/R/get_eurostat_codelist.R
@@ -93,18 +93,18 @@ get_eurostat_codelist <- function(id,
           }
         } else {
           tryCatch({utils::download.file(cls_endpoint,temp,dmethod,quiet=TRUE)},
-                   error = function(e) {
+                   error = function(e) {message("There is an error by the download of the codelist file. Run the same command with verbose=TRUE option to get more info on the issue.")
                    },
-                   warning = function(w) {
+                   warning = function(w) {message("There is a warning by the download of the codelist file. Run the same command with verbose=TRUE option to get more info on the issue.")
                      tbc<-FALSE
                      cls_xml<-NULL
                    })
           if (file.size(temp)!=0 & tbc) {
             tryCatch({cls_xml<-xml2::read_xml(temp)},
-                     error = function(e) {
+                     error = function(e) {message("There is an error by the reading of the downloaded codelist file. Run the same command with verbose=TRUE option to get more info on the issue.")
                        cls_xml<-NULL
                      },
-                     warning = function(w) {
+                     warning = function(w) {message("There is a warning by the reading of the downloaded codelist file. Run the same command with verbose=TRUE option to get more info on the issue.")
                        cls_xml<-NULL
                      })
           } else {

diff --git a/R/get_eurostat_data.R b/R/get_eurostat_data.R
@@ -63,7 +63,7 @@
 #' 
 #' @details Data sets are downloaded from the Eurostat Web Services 
 #' \href{https://wikis.ec.europa.eu/pages/viewpage.action?pageId=44165555}{SDMX API} if there is a filter otherwise the 
-#' \href{https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing}{the Eurostat bulk download facility} is used.
+#' \href{https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API}{the Eurostat bulk download facility} is used.
 #' If only the table \code{id} is given, the whole table is downloaded from the
 #' bulk download facility. If also \code{filters} or \code{date_filter} is defined then the SDMX REST API is
 #' used. In case after filtering the dataset has more rows than the limitation of the SDMX REST API (1 million values at one time) then the bulk download is used to retrieve the whole dataset .
@@ -394,19 +394,23 @@ get_eurostat_data <- function(id,
                 tryCatch({rdat<-data.table::fread(text=readLines(gzcon(url(x))),sep=',',sep2=',',colClasses='character',header=TRUE,stringsAsFactors=stringsAsFactors)},
                          error = function(e) {
                            if (verbose){message("get_eurostat_data - Error by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(e),collapse="\n"))}
+                           else {message("There is an error by the reading of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")}
                            tbc<-FALSE
                          },
                          warning = function(w) {
                            if (verbose){message("get_eurostat_data - Warning by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(w),collapse="\n"))}
+                           else {message("There is a warning by the reading of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")}
                          })
               } else{
                 tryCatch({rdat<-data.table::fread(paste(readLines(gzcon(url(x))),collapse="\n"),sep=',',sep2=',',colClasses='character',header=TRUE,stringsAsFactors=stringsAsFactors)},
                          error = function(e) {
                            if (verbose){message("get_eurostat_data - Error by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(e),collapse="\n"))}
+                           else {message("There is an error by the reading of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")}
                            tbc<-FALSE
                          },
                          warning = function(w) {
                            if (verbose){message("get_eurostat_rdat - Warning by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(w),collapse="\n"))}
+                           else {message("There is a warning by the reading of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")}
                          })
               }
               if(!is.null(rdat)){
@@ -432,10 +436,12 @@ get_eurostat_data <- function(id,
               tryCatch({utils::download.file(x,temp,dmethod,quiet=!verbose)},
                        error = function(e) {
                          if (verbose) {message("get_eurostat_data - Error by the download the xml file:",'\n',paste(unlist(e),collapse="\n"))}
+                         else {message("There is an error by the download of the XML file. Run the same command with verbose=TRUE option to get more info on the issue.")}
                          tbc<-FALSE
                        },
                        warning = function(w) {
                          if(verbose){message("get_eurostat_data - Warning by the download the xml file:",'\n',paste(unlist(w),collapse="\n"))}
+                         else {message("There is a warning by the download of the XML file. Run the same command with verbose=TRUE option to get more info on the issue.")}
                          tbc<-FALSE
                        })
               if (length(temp)==0) {tbc<-FALSE}
@@ -444,10 +450,12 @@ get_eurostat_data <- function(id,
                 tryCatch({xml_foot<-xml2::xml_find_all(xml2::read_xml(temp),".//footer:Message")},
                          error = function(e) {
                            if (verbose) {message("get_eurostat_data - Error by the extraction of the footer from the xml:",'\n',paste(unlist(e),collapse="\n"))}
+                           else {message("There is an error by the reading of the downloaded XML file. Run the same command with verbose=TRUE option to get more info on the issue.")}
                            tbc<-FALSE
                          },
                          warning = function(w) {
                            if(verbose){message("get_eurostat_data - Warning by the extraction of the footer from the xml:",'\n',paste(unlist(w),collapse="\n"))}
+                           else {message("There is a warning by the reading of the downloaded XML file. Run the same command with verbose=TRUE option to get more info on the issue.")}
                            tbc<-FALSE
                          })
               } 
@@ -469,11 +477,28 @@ get_eurostat_data <- function(id,
                 }
               } else {
                 tbc<-FALSE
+                tryCatch({xml_fault<-xml2::xml_find_all(xml2::read_xml(temp),".//S:Fault")},
+                        error = function(e) {
+                              if (verbose) {message("get_eurostat_data - Error by the extraction of the faultcode from the xml:",'\n',paste(unlist(e),collapse="\n"))}
+                              },
+                        warning = function(w) {
+                                if(verbose){message("get_eurostat_data - Warning by the extraction of the faultcode from the xml:",'\n',paste(unlist(w),collapse="\n"))}
+                              })
                 message("Problem by the extraction of the footer information from the xml_file.")
               }
               if(tbc){
-                tryCatch({xml_mark<-switch(rav,"1" = ".//generic:Series","2" = ".//g:Series")
-                xml_leafs<-xml2::xml_find_all(xml2::read_xml(temp),xml_mark)
+               xml_mark<-switch(rav,"1" = ".//generic:Series","2" = ".//g:Series")
+               tryCatch({xml_leafs<-xml2::xml_find_all(xml2::read_xml(temp),xml_mark)},
+                         error = function(e) {
+                                if (verbose) {message("get_eurostat_data - Error by the reading of data from the downloaded XML:",'\n',paste(unlist(e),collapse="\n"))}
+                                else {message("There is an error by the reading of the downloaded XML file. Run the same command with verbose=TRUE option to get more info on the issue.")}
+                         },
+                         warning = function(w) { 
+                           if(verbose){message("get_eurostat_data - Warning by reading of data from the downloaded XML file:",'\n',paste(unlist(w),collapse="\n"))}
+                           else {message("There is a warning by the downloaded of the XML file. Run the same command with verbose=TRUE option to get more info on the issue.")}
+                           }
+                )
+               tryCatch({           
                 if (verbose) {message(class(xml_leafs),"\nnumber of nodes: ",length(xml_leafs),"\nnumber of cores: ",getOption("restatapi_cores",1L),"\n")}
                 if (Sys.info()[['sysname']]=='Windows'){
                   if (getOption("restatapi_cores",1L)==1) {
@@ -491,7 +516,7 @@ get_eurostat_data <- function(id,
                   rdat<-data.table::rbindlist(parallel::mclapply(xml_leafs,extract_data,keep_flags=keep_flags,stringsAsFactors=stringsAsFactors,bulk=FALSE,mc.cores=getOption("restatapi_cores",1L)))                                  
                 }
                 },
-                error = function(e){rdat<-NULL},
+                error = function(e){if (verbose){message("get_eurostat_data - ",e);rdat<-NULL}},
                 warning = function(w){if (verbose){message("get_eurostat_data - ",w)}}
                 )
               }

diff --git a/R/get_eurostat_dsd.R b/R/get_eurostat_dsd.R
@@ -104,18 +104,18 @@ get_eurostat_dsd <- function(id,
         }
       } else {
         tryCatch({utils::download.file(dsd_endpoint,temp,dmethod,quiet=TRUE)},
-                 error = function(e) {
+                 error = function(e) {message("There is an error by the download of the DSD file. Run the same command with verbose=TRUE option to get more info on the issue.")
                  },
-                 warning = function(w) {
+                 warning = function(w) {message("There is a warning by the download of the DSD file. Run the same command with verbose=TRUE option to get more info on the issue.")
                    tbc<-FALSE
                    dsd_xml<-NULL
                  })
         if (file.size(temp)!=0 & tbc) {
           tryCatch({dsd_xml<-xml2::read_xml(temp)},
-                 error = function(e) {
+                 error = function(e) {message("There is an error by the reading of the downloaded DSD file. Run the same command with verbose=TRUE option to get more info on the issue.")
                    dsd_xml<-NULL
                  },
-                 warning = function(w) {
+                 warning = function(w) {message("There is an error by the reading of the downloaded DSD file. Run the same command with verbose=TRUE option to get more info on the issue.")
                    dsd_xml<-NULL
                  })
         } else {
@@ -170,7 +170,7 @@ get_eurostat_dsd <- function(id,
                        cc_xml<-NULL
                      })
             if (file.size(temp)!=0 & tbc) {
-              message("Trying to extract the CC from: ",temp)
+              message("get_eurostat_dsd - Trying to extract the CC from: ",temp)
               tryCatch({cc_xml<-xml2::read_xml(temp)},
                        error = function(e) {
                          message("get_eurostat_dsd - Error during the extraction of the XML from the downloaded CC file:",'\n',paste(unlist(e),collapse="\n"))
@@ -184,18 +184,21 @@ get_eurostat_dsd <- function(id,
             }
           } else {
             tryCatch({utils::download.file(cc_endpoint,temp,dmethod,quiet=TRUE)},
-                     error = function(e) {
+                     error = function(e) {message("There is an error by the download of the content constraint file. Run the same command with verbose=TRUE option to get more info on the issue.")
+                       tbc<-FALSE
+                       cc_xml<-NULL
                      },
-                     warning = function(w) {
+                     warning = function(w) {message("There is an error by the download of the content constraint file. Run the same command with verbose=TRUE option to get more info on the issue.")
                        tbc<-FALSE
                        cc_xml<-NULL
                      })
             if (file.size(temp)!=0 & tbc) {
               tryCatch({cc_xml<-xml2::read_xml(temp)},
-                       error = function(e) {
+                       error = function(e) {message("There is an error by the reading of the downloaded CC file. Run the same command with verbose=TRUE option to get more info on the issue.")
                          cc_xml<-NULL
                        },
-                       warning = function(w) {
+                       warning = function(w) {message("There is a warning by the reading of the downloaded CC file. Run the same command with verbose=TRUE option to get more info on the issue.")
+                         cc_xml<-NULL
                        })
             } else {
               cc_xml<-NULL