#update parsers

qapro-rs/Cargo.toml

@@ -52,12 +52,31 @@ futures-test = "^0.3"
 futures-util = "^0.3"
 async-trait= "0.1.52"
 chrono-tz= { version = "0.5.3", features = ["serde"] }
+chrono = { version = "0.4.19", features = ["serde"] }
 
-chrono = "0.4.19"
 thiserror = "1.0"
 rand = "0.7"
 quadprog = "0.0.1"
 itertools = "0.10.3"
+
+
+##parsers
+
+parse-display = "0.5.0"
+quick-xml = { version = "0.22.0", features = ["serialize"] }
+serde_yaml = "0.8.17"
+serde_partiql = "1.1.64"
+structopt = { version = "0.3.21", optional = true }
+anyhow = "1.0.40"
+atty = "0.2.14"
+bat = { version = "0.18.1", optional = true }
+collect-mac = "0.1.0"
+indexmap = { version = "1.6.2", features = ["serde"] }
+
+nom = "6.1.2"
+ordered-float = { version = "2.0", default-features = false, features = [
+    "serde",
+] }
 #arrow = {version='6.4.0',default-features = true,   features = ["csv", "prettyprint", "ipc"] }
 #
 #datafusion = {version='6.0.0'}
@@ -66,7 +85,7 @@ evalexpr = "6.6.0"
 [dependencies.polars]
 #version = "0.18.0"
 git = "https://github.com/yutiansut/polars.git"
-rev = "926a57c874d224119b124bc500be97d6ddb7fc7f"
+rev = "16ab34c5d2d1015354577aa473103b1acb5989a8"
 features = [
     "zip_with",
     "strings",

qapro-rs/examples/factorplayground.rs

@@ -8,11 +8,11 @@ use polars::prelude::*;
 use polars::series::ops::NullBehavior;
 
 use itertools::izip;
+use qapro_rs::qaaccount::account::QA_Account;
 use qapro_rs::qaconnector::clickhouse::ckclient;
 use qapro_rs::qaconnector::clickhouse::ckclient::DataConnector;
 use qapro_rs::qadatastruct::stockday::QADataStruct_StockDay;
 use qapro_rs::qaenv::localenv::CONFIG;
-use qapro_rs::qaaccount::account::QA_Account;
 
 #[actix_rt::main]
 async fn main() {
@@ -162,7 +162,6 @@ async fn main() {
         }
     }
 
-
     println!("calc get row time {:#?}", sw.elapsed());
     acc.to_csv("".to_string());
 }

qapro-rs/readme.md

@@ -33,6 +33,8 @@ cargo run --release example.toml
 cargo run --example api --release example.toml [开启的服务在example.toml中配置 默认5000端口]
 
 
+
+
 @yutiansut
 
 2021-12-22

qapro-rs/src/lib.rs

@@ -1,3 +1,4 @@
+#![feature(box_patterns)]
 pub mod qaaccount;
 pub mod qaconnector;
 pub mod qadata;
@@ -17,3 +18,5 @@ pub mod qastrategy;
 pub mod qadatastruct;
 pub mod qafactor;
 pub mod qahandlers;
+
+pub mod parsers;

qapro-rs/src/parsers/engine.rs

@@ -0,0 +1,44 @@
+use std::str::FromStr;
+
+use crate::parsers::lang::{Lang, LangType};
+use crate::parsers::planner;
+use crate::parsers::sql::Sql;
+use crate::parsers::value::PqlValue;
+
+pub fn evaluate(sql: &str, input: &str, from: &str, to: &str) -> anyhow::Result<String> {
+    let from_lang_type = LangType::from_str(&from)?;
+    let to_lang_type = LangType::from_str(&to)?;
+    let mut lang = Lang::from_as(&input, from_lang_type)?;
+
+    let sql = Sql::from_str(&sql)?;
+
+    let result = planner::evaluate(sql, lang.data);
+    lang.to = to_lang_type;
+    lang.data = result;
+    let output = lang.to_string(true)?;
+
+    Ok(output)
+}
+
+pub fn loads(input: &str, from: &str) -> anyhow::Result<PqlValue> {
+    let from_lang_type = LangType::from_str(&from)?;
+    let lang = Lang::from_as(&input, from_lang_type)?;
+    let value = lang.data;
+    Ok(value)
+}
+
+pub fn dumps(data: PqlValue, to: &str) -> anyhow::Result<String> {
+    let to_lang_type = LangType::from_str(&to)?;
+    let mut lang = Lang::default();
+    lang.data = data;
+    lang.to = to_lang_type;
+    let output = lang.to_string(true)?;
+    Ok(output)
+}
+
+pub fn query_evaluate(data: PqlValue, sql: &str) -> anyhow::Result<PqlValue> {
+    let sql = Sql::from_str(&sql)?;
+    let data = PqlValue::from(data);
+    let value = planner::evaluate(sql, data);
+    Ok(value)
+}

qapro-rs/src/parsers/lang.rs

@@ -0,0 +1,214 @@
+use std::str::FromStr;
+
+use parse_display::{Display, FromStr};
+
+#[cfg(feature = "table")]
+use polars::prelude::CsvReader;
+#[cfg(feature = "table")]
+use polars::prelude::*;
+
+use crate::parsers::value::{BPqlValue, PqlValue, TomlValue};
+
+#[derive(Display, FromStr, PartialEq, Clone, Debug)]
+#[display(style = "snake_case")]
+pub enum LangType {
+    Json,
+    Yaml,
+    Toml,
+    Xml,
+    #[cfg(feature = "table")]
+    Csv,
+}
+
+impl Default for LangType {
+    fn default() -> Self {
+        Self::Json
+    }
+}
+
+#[derive(Debug, Default, Clone, PartialEq)]
+pub struct Lang {
+    pub data: PqlValue,
+    pub text: String,
+    pub from: LangType,
+    pub to: LangType,
+    pub colnames: Vec<String>,
+}
+
+impl FromStr for Lang {
+    type Err = anyhow::Error;
+
+    fn from_str(input: &str) -> anyhow::Result<Self> {
+        if let Ok(this) = Self::from_as_json(input) {
+            Ok(this)
+        } else if let Ok(this) = Self::from_as_toml(input) {
+            Ok(this)
+        } else if let Ok(this) = Self::from_as_xml(input) {
+            Ok(this)
+        } else if let Ok(this) = Self::from_as_xml(input) {
+            Ok(this)
+        } else {
+            anyhow::bail!("not supported")
+        }
+    }
+}
+
+impl Lang {
+    pub fn from_as(input: &str, lnag_type: LangType) -> anyhow::Result<Self> {
+        match lnag_type {
+            #[cfg(feature = "table")]
+            LangType::Csv => Self::from_as_csv(input),
+            LangType::Json => Self::from_as_json(input),
+            LangType::Toml => Self::from_as_toml(input),
+            LangType::Yaml => Self::from_as_yaml(input),
+            LangType::Xml => Self::from_as_xml(input),
+        }
+    }
+
+    #[cfg(feature = "table")]
+    pub fn from_as_csv(input: &str) -> anyhow::Result<Self> {
+        if let Ok(data) = csvstr_to_pqlv(input) {
+            Ok(Self {
+                data,
+                text: input.to_string(),
+                from: LangType::Csv,
+                to: LangType::Csv,
+                colnames: Vec::default(),
+            })
+        } else {
+            anyhow::bail!("fail to parse input as csv");
+        }
+    }
+
+    pub fn from_as_json(input: &str) -> anyhow::Result<Self> {
+        if let Ok(data) = serde_json::from_str::<serde_json::value::Value>(input) {
+            // Json does not distinguish between Float and Int. For this reason, it it parsed once with serde_json::value::Value, not crate::value::PqlValue.
+            Ok(Self {
+                data: crate::parsers::value::json_value::to_pqlvalue(data),
+                text: input.to_string(),
+                from: LangType::Json,
+                to: LangType::Json,
+                colnames: Vec::default(),
+            })
+        } else {
+            anyhow::bail!("fail to parse input as json");
+        }
+    }
+
+    pub fn from_as_toml(input: &str) -> anyhow::Result<Self> {
+        if let Ok(data) = toml::from_str::<PqlValue>(input) {
+            Ok(Self {
+                data,
+                text: input.to_string(),
+                from: LangType::Toml,
+                to: LangType::Toml,
+                colnames: Vec::default(),
+            })
+        } else {
+            anyhow::bail!("fail to parse input as toml");
+        }
+    }
+
+    pub fn from_as_xml(input: &str) -> anyhow::Result<Self> {
+        if let Ok(data) = quick_xml::de::from_str::<PqlValue>(input) {
+            Ok(Self {
+                data,
+                text: input.to_string(),
+                from: LangType::Xml,
+                to: LangType::Xml,
+                colnames: Vec::default(),
+            })
+        } else {
+            anyhow::bail!("fail to parse input as xml");
+        }
+    }
+
+    pub fn from_as_yaml(input: &str) -> anyhow::Result<Self> {
+        if let Ok(data) = serde_yaml::from_str::<PqlValue>(input) {
+            Ok(Self {
+                data,
+                text: input.to_string(),
+                from: LangType::Yaml,
+                to: LangType::Yaml,
+                colnames: Vec::default(),
+            })
+        } else {
+            anyhow::bail!("fail to parse input as yaml");
+        }
+    }
+
+    pub fn sort_keys(&mut self) {
+        let json = serde_json::to_string(&self.data).unwrap();
+        let bdata = serde_json::from_str::<BPqlValue>(&json).unwrap();
+        let bjson = serde_json::to_string(&bdata).unwrap();
+        let data = serde_json::from_str::<PqlValue>(&bjson).unwrap();
+        self.data = data;
+    }
+
+    pub fn to_string(&self, compact: bool) -> anyhow::Result<String> {
+        let output = match (&self.to, &self.from == &self.to) {
+            #[cfg(feature = "table")]
+            (LangType::Csv, _) => {
+                // To pad missing values with null, serialize them to json, deserialize them with polars, and write them to csv from there.
+                let sss = match &self.data {
+                    PqlValue::Array(array) => array
+                        .iter()
+                        .map(|v| serde_json::to_string(&v).unwrap())
+                        .collect::<Vec<String>>()
+                        .join("\n"),
+                    _ => anyhow::bail!("must array"),
+                };
+                let c = std::io::Cursor::new(&sss);
+                let mut df = JsonReader::new(c).infer_schema(Some(100)).finish()?;
+
+                let mut v = Vec::new();
+                CsvWriter::new(&mut v)
+                    .has_headers(true)
+                    .with_delimiter(b',')
+                    .finish(&mut df)?;
+                let s = String::from_utf8(v)?;
+                s
+            }
+            (LangType::Json, _) if compact => serde_json::to_string(&self.data).unwrap(),
+            (LangType::Json, _) => serde_json::to_string_pretty(&self.data).unwrap(),
+            (_, true) => self.text.to_owned(),
+            (LangType::Toml, _) => {
+                let v = TomlValue::from(self.data.to_owned());
+                toml::to_string_pretty(&v).unwrap()
+            }
+            (LangType::Yaml, _) => serde_yaml::to_string(&self.data)
+                .unwrap()
+                .trim_start_matches("---\n")
+                .to_string(),
+            (LangType::Xml, _) => quick_xml::se::to_string(&self.data).unwrap(),
+        };
+
+        Ok(output)
+    }
+
+    #[cfg(feature = "cli")]
+    pub fn print(&self, compact: bool) -> anyhow::Result<()> {
+        let output = self.to_string(compact)?;
+
+        if atty::is(atty::Stream::Stdout) {
+            let bytes = output.as_bytes().to_vec();
+            let lang_type = self.to.to_string();
+
+            bat::PrettyPrinter::new()
+                .language(&lang_type)
+                .input(bat::Input::from_bytes(&bytes))
+                .print()
+                .unwrap();
+        } else {
+            println!("{}", &output);
+        }
+        Ok(())
+    }
+}
+
+#[cfg(feature = "table")]
+fn csvstr_to_pqlv(input: &str) -> anyhow::Result<PqlValue> {
+    let c = std::io::Cursor::new(input.to_owned());
+    let df = CsvReader::new(c).infer_schema(Some(100)).finish()?;
+    Ok(PqlValue::from(df))
+}

qapro-rs/src/parsers/mod.rs

@@ -0,0 +1,10 @@
+#![feature(box_patterns)]
+pub mod engine;
+pub mod lang;
+pub mod models;
+pub mod parser;
+pub mod planner;
+pub mod pqlir_parser;
+pub mod sql;
+pub mod utils;
+pub mod value;