update of debug messages

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: restatapi
 Type: Package
 Title: Search and Retrieve Data from Eurostat Database
-Date: 2024-01-3
-Version: 0.22.4
+Date: 2024-01-07
+Version: 0.22.5
 Encoding: UTF-8
 Authors@R: c(person("Mátyás", "Mészáros", email = "[email protected]", role = c("aut", "cre")),
             person("Sebastian", "Weinand", role = "ctb"))

NEWS.md

@@ -1,3 +1,7 @@
+# restatapi 0.22.5
+
+- synchronization of debug messages 
+
 # restatapi 0.22.4
 
 - updating tests because of missing dataset in the XML version of the Table of Contents 

R/create_filter_table.R

@@ -74,7 +74,7 @@ create_filter_table <- function(filters,date_filter=FALSE,dsd=NULL,exact_match=T
   if (!date_filter & is.null(dsd)){
     message("The DSD is missing from the create_filter_table function.")
   } else{
-    if (verbose) {message("\ncreate_filter_table - filters class: ",class(filters),"; size: ",length(filters),"; filters:",filters)}
+    if (verbose) {message("create_filter_table - filters class: ",class(filters),"; size: ",length(filters),"; filters:",filters)}
     if (inherits(filters,"name")) {
       try(filters<-local(filters),silent=verbose)
     }  
@@ -91,7 +91,7 @@ create_filter_table <- function(filters,date_filter=FALSE,dsd=NULL,exact_match=T
       } else {
         df<-as.character(filters)
       }
-      if (verbose) {message("length df: ",length(df)," -*- df: ",paste(df,collapse=", "))}
+      if (verbose) {message("create_filter_table - length df: ",length(df)," content df: ",paste(df,collapse=", "))}
       if (df[1]=="c"){
         df<-df[2:length(df)]
       } else {
@@ -114,7 +114,7 @@ create_filter_table <- function(filters,date_filter=FALSE,dsd=NULL,exact_match=T
               res<-list(sd=min(dates),ed=max(dates))
             } else{
               res<-NULL
-              if(verbose){message(paste0("Could not parse date filter: '",paste0(dates,collapse=":"),"' (at least one date not in yyyy[-mm][-dd] format or incorrect date value). The date filter is ignored."))}
+              if(verbose){message(paste0("create_filter_table - Could not parse date filter: '",paste0(dates,collapse=":"),"' (at least one date not in yyyy[-mm][-dd] format or incorrect date value). The date filter is ignored."))}
             }
           } else if (grepl("<|>",sdf,perl=TRUE)){
             if(check_tf(gsub("<|>","",sdf,perl=TRUE),time_formats)){
@@ -124,18 +124,18 @@ create_filter_table <- function(filters,date_filter=FALSE,dsd=NULL,exact_match=T
                 res<-list(sd=gsub("<|>","",sdf,perl=TRUE),ed=Inf)
               } else {
                 res<-NULL
-                if (verbose) {message(paste0("Could not parse date filter: '", sdf,"' not in [<>]yyyy[-mm][-dd][<>] format or incorrect date value. The date filter is ignored."))}
+                if (verbose) {message(paste0("create_filter_table - Could not parse date filter: '", sdf,"' not in [<>]yyyy[-mm][-dd][<>] format or incorrect date value. The date filter is ignored."))}
               }
             } else {
               res<-NULL
-              if(verbose){message(paste0("Could not parse date filter: '",sdf,"' (not in yyyy[-mm][-dd] format or incorrect date value. The date filter is ignored."))}
+              if(verbose){message(paste0("create_filter_table - Could not parse date filter: '",sdf,"' (not in yyyy[-mm][-dd] format or incorrect date value. The date filter is ignored."))}
             }
           } else{
             if(check_tf(sdf,time_formats)){
               res<-list(sd=sdf,ed=sdf)  
             } else {
               res<-NULL
-              if (verbose) {message(paste0("Could not parse date filter: '",sdf,"' not in [<>]yyyy[-mm][-dd][<>] format or incorrect date value. The date filter is ignored."))}
+              if (verbose) {message(paste0("create_filter_table - Could not parse date filter: '",sdf,"' not in [<>]yyyy[-mm][-dd][<>] format or incorrect date value. The date filter is ignored."))}
             }
           }  
         }

R/get_compressed_sdmx.R

@@ -26,7 +26,7 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){
     return(NULL)
   } else {
     if (tbc) {
-      if (verbose) message("get_compressed_sdmx - url:",url,"\nget_compressed_sdmx - format:",format)
+      if (verbose) message("\nget_compressed_sdmx - url:",url,"\nget_compressed_sdmx - format:",format)
 
       if (format=="zip"){ # zip format
         # download file 

R/get_eurostat_bulk.R

@@ -112,7 +112,7 @@ get_eurostat_bulk <- function(id,
       load_cfg()
     }  
   }
-  if (verbose)  {message("get_eurostat_bulk - API version:",get("rav",envir=restatapi::.restatapi_env))}
+  # if (verbose)  {message("get_eurostat_bulk - API version:",get("rav",envir=restatapi::.restatapi_env))}
 
   if (!is.null(id)){id<-tolower(trimws(id))} else {
     tbc<-FALSE
@@ -128,7 +128,9 @@ get_eurostat_bulk <- function(id,
       } else {
         if (any(grepl(id,toc$code,ignore.case=TRUE))){
           udate<-toc$lastUpdate[grepl(id,toc$code,ignore.case=TRUE)]
-          if (verbose) {message("get_eurostat_bulk - bulk TOC rows: ",nrow(toc),"\nbulk url: ",toc$downloadLink.tsv[grepl(id,toc$code,ignore.case=TRUE)],"\ndata rowcount: ",toc$values[grepl(id,toc$code,ignore.case=TRUE)])}
+          if (verbose) {message("get_eurostat_bulk - TOC rows: ",nrow(toc),"
+                                \nget_eurostat_bulk - bulk url: ",toc$downloadLink.tsv[grepl(id,toc$code,ignore.case=TRUE)],
+                                "\nget_eurostat_bulk - ndata rowcount in TOC: ",toc$values[grepl(id,toc$code,ignore.case=TRUE)])}
         } else {
           message(paste0("'",id,"' is not in the table of contents. Please check if the 'id' is correctly spelled."))
           tbc<-FALSE
@@ -146,7 +148,9 @@ get_eurostat_bulk <- function(id,
     }
 
     if ((!cache)|is.null(restat_bulk)|(update_cache)){
-      if (verbose) {message("get_eurostat_bulk - ", class(id),"txt",class(cache),class(update_cache),class(cache_dir),class(compress_file),class(stringsAsFactors),class(keep_flags),class(check_toc),class(melt),class(verbose))}
+      if (verbose) {message("get_eurostat_bulk - class of id, cache, update_cache, cache_dir, compress_file, stringsAsFactors, keep_flags, check_toc, melt, verbose:\n", class(id)," - ",class(cache)," -",class(update_cache),
+                            " - ",class(cache_dir)," - ",class(compress_file)," - ",class(stringsAsFactors)," - ",class(keep_flags),
+                            " - ",class(check_toc)," - ",class(melt)," - ",class(verbose))}
       restat_bulk<-restatapi::get_eurostat_raw(id,"txt",cache,update_cache,cache_dir,compress_file,stringsAsFactors,keep_flags,check_toc,melt=TRUE,verbose=verbose)
     }
   }  
@@ -206,7 +210,7 @@ get_eurostat_bulk <- function(id,
   if ((!is.null(restat_bulk))&cache&(all(!grepl("get_eurostat_data",as.character(sys.calls()),perl=TRUE)))){
     oname<-paste0("b_",id,"-",udate,"-",sum(keep_flags),"-",sum(cflags),sub("-$","",paste0("-",select_freq),perl=TRUE))
     pl<-restatapi::put_eurostat_cache(restat_bulk,oname,update_cache,cache_dir,compress_file)
-    if ((!is.null(pl))&(verbose)) {message("get_eurostat_bulk - The bulk data was cached ",pl,".\n" )}
+    if ((!is.null(pl))&(verbose)) {message("get_eurostat_bulk - The bulk data was cached ",pl,"." )}
   }
 
   return(restat_bulk)

R/get_eurostat_data.R

@@ -193,7 +193,7 @@ get_eurostat_data <- function(id,
   verbose<-verbose|getOption("restatapi_verbose",FALSE)
   update_cache<-update_cache|getOption("restatapi_update",FALSE)
   dmethod<-getOption("restatapi_dmethod",get("dmethod",envir=restatapi::.restatapi_env))
-  if (verbose)  {message("\nget_eurostat_data - API version:",get("rav",envir=restatapi::.restatapi_env))}
+  # if (verbose)  {message("\nget_eurostat_data - API version:",get("rav",envir=restatapi::.restatapi_env))}
   tbc<-cr<-TRUE # to be continued for the next steps  / cache result data.table 
   if (verbose) {message("get_eurostat_data - footer code option value at start:",paste(getOption("code_opt",NULL),collapse=", "))}
   options(code_opt=NULL)
@@ -223,7 +223,7 @@ get_eurostat_data <- function(id,
     check_toc<-FALSE
     message("The dataset 'id' is missing.")
   }
-  if (verbose) {message("get_eurostat_data - id:",id)}
+  if (verbose) {message("get_eurostat_data - id: ",id)}
 
   if (check_toc){
     toc<-restatapi::get_eurostat_toc(verbose=verbose)
@@ -244,23 +244,23 @@ get_eurostat_data <- function(id,
   }
 
   if (tbc){
-    if (verbose) {message("get_eurostat_data - select_freq:",select_freq)}
+    if (verbose) {message("get_eurostat_data - select_freq: ",select_freq)}
     if(!is.null(select_freq)){ 
-      if (verbose) {message("get_eurostat_data - not NULL select_freq:",select_freq)}
+      if (verbose) {message("get_eurostat_data - not NULL select_freq: ",select_freq)}
       append_sf<-FALSE
       if (is.null(filters)|(length(filters)>1)) # no filter or there is already several filters defined => the select_freq is appended to the the filters
       {
-        if (verbose) {message("get_eurostat_data - select_freq with NULL filters:",select_freq)}
+        if (verbose) {message("get_eurostat_data - select_freq with NULL filters: ",select_freq)}
         append_sf<-TRUE  
       } else if (!is.null(filters)) #there are filters
       {  
-        if (verbose) {message("get_eurostat_data - select_freq with filters:",select_freq)}
+        if (verbose) {message("get_eurostat_data - select_freq with filters: ",select_freq)}
         if (length(filters)==1) #there is a single string filter
         {
-          if (verbose) {message("get_eurostat_data - select_freq with 1 filter:",select_freq)}
+          if (verbose) {message("get_eurostat_data - select_freq with 1 filter: ",select_freq)}
           if (grepl("\\.",filters,perl=TRUE)) #filter is given as a string for the REST API
           { 
-            if (verbose) {message("get_eurostat_data - select_freq with string filter with '.'at the beginning:",select_freq)}
+            if (verbose) {message("get_eurostat_data - select_freq with string filter with '.'at the beginning: ",select_freq)}
             if (grepl("^\\.",filters,perl=TRUE)) # no FREQ value is given in the filter
             { 
               filters<-paste0(select_freq,filters)
@@ -271,7 +271,7 @@ get_eurostat_data <- function(id,
             append_sf<-TRUE
           }
         } else {
-          if (verbose) {message("get_eurostat_data - select_freq with more than 1 filter:",select_freq)}
+          if (verbose) {message("get_eurostat_data - select_freq with more than 1 filter: ",select_freq)}
           append_sf<-TRUE
         }
       }
@@ -499,7 +499,9 @@ get_eurostat_data <- function(id,
                            }
                 )
                tryCatch({           
-                if (verbose) {message(class(xml_leafs),"\nnumber of nodes: ",length(xml_leafs),"\nnumber of cores: ",getOption("restatapi_cores",1L),"\n")}
+                if (verbose) {message("get_eurostat_data - class(xml_leafs):",class(xml_leafs),
+                                      "\nget_eurostat_data - number of nodes: ",length(xml_leafs),
+                                      "\nget_eurostat_data - number of cores: ",getOption("restatapi_cores",1L))}
                 if (Sys.info()[['sysname']]=='Windows'){
                   if (getOption("restatapi_cores",1L)==1) {
                     if (verbose) message("No parallel")
@@ -731,7 +733,7 @@ get_eurostat_data <- function(id,
       }
     }  
   }
-  if (getOption("restatapi_verbose",FALSE))  {message("get_eurostat_data - restat NULL:",is.null(restat),"\n\t id",id,"\n\t filters",filters_url,"\n\t date filter",date_filter)}
+  if (getOption("restatapi_verbose",FALSE))  {message("get_eurostat_data - restat NULL?:",is.null(restat),"\n\t id: ",id,"\n\t filters: ",filters_url,"\n\t date filter: ",date_filter)}
   return(restat)
 }
 

R/get_eurostat_dsd.R

@@ -44,7 +44,7 @@ get_eurostat_dsd <- function(id,
                              verbose=FALSE,...) {
   verbose<-verbose|getOption("restatapi_verbose",FALSE)
   dmethod<-getOption("restatapi_dmethod",get("dmethod",envir=restatapi::.restatapi_env))
-  if (verbose)  {message("\nget_eurostat_dsd - API version:",get("rav",envir=restatapi::.restatapi_env))}
+  # if (verbose)  {message("\nget_eurostat_dsd - API version:",get("rav",envir=restatapi::.restatapi_env))}
   tbc<-TRUE #to be continued for the next steps
   if (is.null(id)){
     warning('No dataset id were provided.')
@@ -71,7 +71,7 @@ get_eurostat_dsd <- function(id,
     if ((!cache)|(is.null(dsd))|(update_cache)){
       cfg<-get("cfg",envir=restatapi::.restatapi_env) 
       rav<-get("rav",envir=restatapi::.restatapi_env)
-      if (verbose)  {message("get_eurostat_dsd - API version:",rav)}
+      if (verbose)  {message("\nget_eurostat_dsd - API version:",rav)}
       dsd_endpoint <- paste0(eval(parse(text=paste0("cfg$QUERY_BASE_URL$'",rav,"'$ESTAT$metadata$'2.1'$datastructure"))),"/", 
                              eval(parse(text=paste0("cfg$QUERY_PRIOR_ID$'",rav,"'$ESTAT$metadata"))),id,"?",
                              eval(parse(text=paste0("cfg$QUERY_PARAMETERS$'",rav,"'$metadata[2]"))),"=",

R/get_eurostat_raw.R

@@ -108,7 +108,7 @@ get_eurostat_raw <- function(id,
   }
   cfg<-get("cfg",envir=restatapi::.restatapi_env) 
   rav<-get("rav",envir=restatapi::.restatapi_env)
-  if (verbose)  {message("get_eurostat_raw - API version:",rav)}
+  # if (verbose)  {message("get_eurostat_raw - API version:",rav)}
   if (!is.null(id)){id<-tolower(trimws(id))} else {
     tbc<-FALSE
     message("The dataset 'id' is missing.")
@@ -141,7 +141,9 @@ get_eurostat_raw <- function(id,
             message("There is no downloadlink in the TOC for ",id)
             tbc<-FALSE
           }
-          if (verbose) {message("get_eurostat_raw - raw TOC rows: ",nrow(toc),"\nbulk url: ",bulk_url,"\ndata rowcount: ",toc$values[grepl(id,toc$code,ignore.case=TRUE)])}
+          if (verbose) {message("get_eurostat_raw - raws of TOC: ",nrow(toc),
+                                "\nget_eurostat_raw - bulk url: ",bulk_url,
+                                "\nget_eurostat_raw - data rowcount in TOC: ",toc$values[grepl(id,toc$code,ignore.case=TRUE)])}
         } else {
           message(paste0("'",id,"' is not in the table of contents. Please check if the 'id' is correctly spelled."))
           tbc<-FALSE
@@ -265,7 +267,7 @@ get_eurostat_raw <- function(id,
                 if (is.character(cname)){
                   cnames<-utils::head(unlist(strsplit(cname,(',|\\\\'))),-1)
                   rname<-switch(rav, "1" = utils::tail(unlist(strsplit(cname,(',|\\\\'))),1),"2"="time")
-                  if (verbose) {message("get_eurostat_raw - class:",class(raw))}
+                  if (verbose) {message("get_eurostat_raw - class(raw): ",class(raw))}
                   data.table::setnames(raw,1,"bdown")
                   raw_melted<-data.table::melt.data.table(raw,"bdown",variable.factor=stringsAsFactors)
                   rm(raw)
@@ -296,13 +298,15 @@ get_eurostat_raw <- function(id,
       } else if (mode=="xml"){
         format<-switch(rav, "1" = "zip", "2" = "gz")
         if (check_toc) {format<-"zip"}
-        if (verbose) {message("get_eurostat_raw - format:",format)}
+        if (verbose) {message("get_eurostat_raw - file format: ",format)}
         sdmx_file<-restatapi::get_compressed_sdmx(bulk_url,verbose=verbose,format=format)
         if(!is.null(sdmx_file)){
           xml_mark<-switch(rav, "1" = ".//data:Series", "2" = ".//Series")
           if (check_toc) {xml_mark<-".//data:Series"}
           xml_leafs<-xml2::xml_find_all(sdmx_file,xml_mark)
-          if (verbose) {message("get_eurostat_raw - ",class(xml_leafs),"\nnumber of nodes: ",length(xml_leafs),"\nnumber of cores: ",getOption("restatapi_cores",1L),"\n")}
+          if (verbose) {message("get_eurostat_raw - class(xml_leafs): ",class(xml_leafs),
+                                "\nget_eurostat_raw - number of nodes: ",length(xml_leafs),
+                                "\nget_eurostat_raw - number of cores: ",getOption("restatapi_cores",1L))}
           if (Sys.info()[['sysname']]=='Windows'){
             if (getOption("restatapi_cores",1L)==1) {
               if (verbose) message("No parallel")
@@ -334,8 +338,10 @@ get_eurostat_raw <- function(id,
       }
       if ((!keep_flags) & ("OBS_STATUS" %in% colnames(restat_raw)))  {restat_raw$OBS_STATUS<-NULL}
     }
-    if (verbose) {message("get_eurostat_raw - caching in raw:",all(!grepl("get_eurostat_bulk|get_eurostat_data",as.character(sys.calls()),perl=TRUE))," local filter:",exists("local_filter",envir=sys.parent(1))," called from:",as.character(sys.call()))}
-    if (verbose) {message("get_eurostat_raw - ", grepl("^get_eurostat_rawidtxt",paste0(as.character(sys.call()),collapse="")))}
+    if (verbose) {message("get_eurostat_raw - caching in raw: ",all(!grepl("get_eurostat_bulk|get_eurostat_data",as.character(sys.calls()),perl=TRUE)),
+                          "\nget_eurostat_raw - local filter: ",exists("local_filter",envir=sys.parent(1)),
+                          "\nget_eurostat_raw - called from: ",as.character(sys.call()),
+                          "\nget_eurostat_raw - get_eurostat_raw in sys.call(): ", grepl("^get_eurostat_rawidtxt",paste0(as.character(sys.call()),collapse="")))}
     #check if the function was called from the get_eurostat_data function
     if (grepl("^get_eurostat_rawidtxt",paste0(as.character(sys.call()),collapse=""))&any(grepl("get_eurostat_data",as.character(sys.calls())))){  
       #if yes get the value of local_filter and force_local_filter from the call

R/get_eurostat_toc.R

@@ -77,7 +77,7 @@ get_eurostat_toc<-function(mode="xml",
       load_cfg()
     }  
   }
-  if (verbose)  {message("get_eurostat_toc - API version:",get("rav",envir=restatapi::.restatapi_env)," - number of cores:",getOption("restatapi_cores",1L))}
+  # if (verbose)  {message("get_eurostat_toc - API version:",get("rav",envir=restatapi::.restatapi_env)," - number of cores:",getOption("restatapi_cores",1L))}
   update_cache<-update_cache|getOption("restatapi_update",FALSE)
   dmethod<-getOption("restatapi_dmethod",get("dmethod",envir=restatapi::.restatapi_env))
   if(any(grepl("get_eurostat_bulk|get_eurostat_data|get_eurostat_raw",as.character(sys.calls()),perl=TRUE))) {update_cache<-FALSE}
@@ -141,7 +141,9 @@ get_eurostat_toc<-function(mode="xml",
       if ((tbc)){
         if (!is.null(xml_leafs)){
           if (length(xml_leafs)>0){
-            if (verbose) {message(class(xml_leafs),"\nnumber of nodes: ",length(xml_leafs),"\nnumber of cores: ",getOption("restatapi_cores",1L),"\n")}
+            if (verbose) {message("get_eurostat_toc - class(xml_leafs): ",class(xml_leafs),
+                                  "\nget_eurostat_toc - number of nodes: ",length(xml_leafs),
+                                  "\nget_eurostat_toc - number of cores: ",getOption("restatapi_cores",1L))}
             if (Sys.info()[['sysname']]=='Windows'){
               if (getOption("restatapi_cores",1L)==1) {
                 if (verbose) message("No parallel")

inst/tinytest/test_restatapi.R

@@ -62,7 +62,7 @@ if (!is.null(xml_toc)){
 r1<-search_eurostat_toc("energy")
 r2<-search_eurostat_toc("energy",ignore.case=TRUE)
 r3<-search_eurostat_toc("energie",lang="de",ignore.case=TRUE)
-if (!is.null(r1)|!is.null(r2)){
+if (!is.null(r1) & !is.null(r2)){
   expect_true(nrow(r1)<nrow(r2)) # 8
 } else {not_checked<-paste(not_checked,8,sep=",")}
 if (!is.null(r3)){