Merge pull request #1727 from jqnatividad/fetch-remove_jsonxf-crate

`fetch` & `fetch post`: remove jsonxf crate; use serde_json to prettify JSON strings

Cargo.toml

@@ -128,7 +128,6 @@ jsonschema = { version = "0.17", features = [
     "resolve-file",
     "resolve-http",
 ], default-features = false }
-jsonxf = { version = "1", optional = true }
 jql-runner = { version = "7.1", default-features = false, optional = true }
 localzone = { version = "0.2", features = ["auto_validation"] }
 log = "0.4"
@@ -294,7 +293,6 @@ fetch = [
     "governor",
     "hashbrown",
     "jql-runner",
-    "jsonxf",
     "redis",
     "serde_stacker",
     "serde_urlencoded",

src/cmd/fetch.rs

@@ -1204,6 +1204,8 @@ fn get_response(
     let mut final_value = String::new();
     let mut api_status;
     let mut api_respheader = HeaderMap::new();
+    let mut api_value = String::new();
+    let mut api_value_json_result: Result<serde_json::Value, serde_json::Error>;
 
     let debug_flag = log_enabled!(Debug);
 
@@ -1230,7 +1232,7 @@ fn get_response(
             // debug!("{resp:?}");
             api_respheader.clone_from(resp.headers());
             api_status = resp.status();
-            let api_value: String = resp.text().unwrap_or_default();
+            api_value = resp.text().unwrap_or_default();
 
             if api_status.is_client_error() || api_status.is_server_error() {
                 error_flag = true;
@@ -1270,16 +1272,29 @@ fn get_response(
                             error_flag = true;
                         },
                     }
-                } else if flag_pretty {
-                    if let Ok(pretty_json) = jsonxf::pretty_print(&api_value) {
-                        final_value = pretty_json;
-                    } else {
-                        final_value = api_value;
-                    }
-                } else if let Ok(minimized_json) = jsonxf::minimize(&api_value) {
-                    final_value = minimized_json;
                 } else {
-                    final_value = api_value;
+                    // validate the JSON response
+                    api_value_json_result = serde_json::from_str::<serde_json::Value>(&api_value);
+                    match api_value_json_result {
+                        Ok(api_value_json) => {
+                            if flag_pretty {
+                                final_value = format!("{api_value_json:#}");
+                            } else {
+                                // use serde_json CompactFormatter to minify the JSON
+                                final_value = format!("{api_value_json}");
+                            }
+                        },
+                        Err(e) => {
+                            error!("json error. json: {api_value:?}, error: {e:?}");
+
+                            if flag_store_error {
+                                final_value = e.to_string();
+                            } else {
+                                final_value = String::new();
+                            }
+                            error_flag = true;
+                        },
+                    }
                 }
             }
         } else {

src/cmd/fetchpost.rs

@@ -1133,6 +1133,8 @@ fn get_response(
     let mut final_value = String::new();
     let mut api_status;
     let mut api_respheader = HeaderMap::new();
+    let mut api_value = String::new();
+    let mut api_value_json_result: Result<serde_json::Value, serde_json::Error>;
 
     let debug_flag = log_enabled!(Debug);
 
@@ -1173,7 +1175,7 @@ fn get_response(
             // debug!("{resp:?}");
             api_respheader.clone_from(resp.headers());
             api_status = resp.status();
-            let api_value: String = resp.text().unwrap_or_default();
+            api_value = resp.text().unwrap_or_default();
 
             if api_status.is_client_error() || api_status.is_server_error() {
                 error_flag = true;
@@ -1213,16 +1215,29 @@ fn get_response(
                             error_flag = true;
                         },
                     }
-                } else if flag_pretty {
-                    if let Ok(pretty_json) = jsonxf::pretty_print(&api_value) {
-                        final_value = pretty_json;
-                    } else {
-                        final_value = api_value;
-                    }
-                } else if let Ok(minimized_json) = jsonxf::minimize(&api_value) {
-                    final_value = minimized_json;
                 } else {
-                    final_value = api_value;
+                    // validate the JSON response
+                    api_value_json_result = serde_json::from_str::<serde_json::Value>(&api_value);
+                    match api_value_json_result {
+                        Ok(api_value_json) => {
+                            if flag_pretty {
+                                final_value = format!("{api_value_json:#}");
+                            } else {
+                                // use serde_json CompactFormatter to minify the JSON
+                                final_value = format!("{api_value_json}");
+                            }
+                        },
+                        Err(e) => {
+                            error!("json error. json: {api_value:?}, error: {e:?}");
+
+                            if flag_store_error {
+                                final_value = e.to_string();
+                            } else {
+                                final_value = String::new();
+                            }
+                            error_flag = true;
+                        },
+                    }
                 }
             }
         } else {

tests/test_fetch.rs

@@ -34,6 +34,79 @@ fn fetch_simple() {
     assert_eq!(got, expected);
 }
 
+#[test]
+// #[ignore = "Temporarily skip this as it seems https://zippopotam.us is not currently available"]
+fn fetch_simple_pretty_json() {
+    let wrk = Workdir::new("fetch_simple_pretty_json");
+    wrk.create(
+        "data.csv",
+        vec![
+            svec!["URL"],
+            svec!["https://api.zippopotam.us/us/99999"],
+            svec!["  http://api.zippopotam.us/us/90210      "],
+            svec!["https://api.zippopotam.us/us/94105"],
+            svec!["http://api.zippopotam.us/us/92802      "],
+            // svec!["https://query.wikidata.org/sparql?query=SELECT%20?dob%20WHERE%20{wd:Q42%20wdt:P569%20?dob.}&format=json"],
+        ],
+    );
+    let mut cmd = wrk.command("fetch");
+    cmd.arg("URL")
+        .arg("data.csv")
+        .arg("--store-error")
+        .arg("--rate-limit")
+        .arg("2")
+        .arg("--pretty")
+        .args(["--new-column", "pretty_response"]);
+
+    let got = wrk.stdout::<String>(&mut cmd);
+
+    let expected = r#"URL,pretty_response
+https://api.zippopotam.us/us/99999,
+http://api.zippopotam.us/us/90210,"{
+  ""post code"": ""90210"",
+  ""country"": ""United States"",
+  ""country abbreviation"": ""US"",
+  ""places"": [
+    {
+      ""place name"": ""Beverly Hills"",
+      ""longitude"": ""-118.4065"",
+      ""state"": ""California"",
+      ""state abbreviation"": ""CA"",
+      ""latitude"": ""34.0901""
+    }
+  ]
+}"
+https://api.zippopotam.us/us/94105,"{
+  ""post code"": ""94105"",
+  ""country"": ""United States"",
+  ""country abbreviation"": ""US"",
+  ""places"": [
+    {
+      ""place name"": ""San Francisco"",
+      ""longitude"": ""-122.3892"",
+      ""state"": ""California"",
+      ""state abbreviation"": ""CA"",
+      ""latitude"": ""37.7864""
+    }
+  ]
+}"
+http://api.zippopotam.us/us/92802,"{
+  ""post code"": ""92802"",
+  ""country"": ""United States"",
+  ""country abbreviation"": ""US"",
+  ""places"": [
+    {
+      ""place name"": ""Anaheim"",
+      ""longitude"": ""-117.9228"",
+      ""state"": ""California"",
+      ""state abbreviation"": ""CA"",
+      ""latitude"": ""33.8085""
+    }
+  ]
+}""#;
+    assert_eq!(got, expected);
+}
+
 #[test]
 // #[ignore = "Temporarily skip this as it seems https://zippopotam.us is not currently available"]
 fn fetch_simple_new_col() {