Initial import

Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "jaxe"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+structopt = "0.3"
+log = "0.4"
+pretty_env_logger = "0.4"
+serde_json = "1"
+termcolor = "1.1"
+nom = "7.1.0"
+anyhow = "1"
+nom_locate = "4.0.0"

Makefile

@@ -0,0 +1,10 @@
+.PHONY: install clean
+
+target/release/jxact: src
+	cargo build --release
+
+install:
+	cargo install --path .
+
+clean:
+	cargo clean

README.md

@@ -0,0 +1,90 @@
+# Jaxe - The J[son] [Pick]axe
+
+jaxe parses [new line delimited json](http://ndjson.org/) on the stdin
+and outputs a color human readable string on stdout.
+
+jaxe supports basic filtering with a simple language. Certain json
+values can be omited or extracted. Invalid json can be displayed in a
+different color or omitted.
+
+
+## Examples
+
+Considering the following newline delimited json log line:
+
+```
+$ cat example.log | jq
+{
+  "logger": "c.a.l.h.logging.RequestLoggingActor",
+  "http_stime": "43",
+  "http_method": "PUT",
+  "http_path": "/api/v1/user",
+  "at": "2022-03-24T08:56:20.576Z",
+  "http_service_name": "reposerver",
+  "msg": "http request",
+  "http_status": "204",
+  "level": "INFO"
+}
+```
+
+
+Piping that line to jaxe will output:
+
+```
+I|2022-03-24T08:56:20.576Z|http_method=PUT http_path=/api/v1/user http_service_name=reposerver http_status=204 http_stime=43 logger=c.a.l.h.logging.RequestLoggingActor msg=http request
+```
+The output is colorized unless you use `-n/--no-colors`:
+
+![screenshot 1](docs/screenshot-01.png)
+
+non json lines will be printed verbatim unless `-j/--no-omit-json` is used.
+
+Often you have fields you don't care about, you can set `JAXE_OMIT` or use `-o/`to filter those fields out:
+
+```
+$ export JAXE_OMIT="logger,http_service_name"
+$ cat example.log | jaxe
+I|2022-03-24T08:56:20.576Z|http_method=PUT http_path=/api/v1/user http_status=204 http_stime=43 msg=http request
+```
+
+A DSL can be used to filter log lines, using `-f/--filter` or `JAXE_FILTER`:
+
+```
+$ cat example.log | jaxe --filter 'http_status==404'
+$
+$ cat example.log | jaxe --filter 'http_status==204'
+I|2022-03-24T08:56:20.576Z|http_method=PUT http_path=/api/v1/user http_status=204 http_stime=43 msg=http request
+$
+$ cat example.log | jaxe --filter 'and(contains(msg,"http request"), not(contains(msg,"username")))'
+```
+
+You can extract only certains values from the json:
+
+```
+$ cat example.log | jaxe --extract http_method,http_status,msg
+I|2022-03-24T08:56:20.576Z|http_method=PUT http_status=204 msg=http request 
+```
+
+### Filtering DSL
+
+The following DSL can be used with `-f/--filter` to filter lines. `.`
+can be used to refer to values nested in json objects.
+
+
+|                   Expression | Example                                            |
+|-----------------------------:|:--------------------------------------------------:|
+| equals/not equals `==`, `!=` | `mykey0.otherkey == myvalue`                       |
+|           `and(exp)/or(exp)` | `and(http_status == 200, http_method != GET)`      |
+|                   `not(exp)` | `not(and(http_status == 200, http_method != GET))` |
+|         `contains(key, str)` | `contains(mykey, somestr)`                         |
+|                `exists(key)` | `exists(mykey)`                                    |
+
+## Configuration
+
+`JAXE_OMIT` and `JAXE_FILTER` can be set the same was as `-o/--omit` and `-f/--filter`.
+
+## Install
+
+Binaries can be downloaded for the releases tab.
+
+

src/cli.rs

@@ -0,0 +1,32 @@
+use std::str::FromStr;
+use anyhow::Result;
+use std::io;
+
+#[derive(Debug)]
+pub (crate) struct MultOpt<T : Sized>(pub(crate) Vec<T>);
+
+impl Default for MultOpt<String> {
+    fn default() -> Self {
+        Self(Vec::new())
+    }
+}
+
+
+impl std::fmt::Display for MultOpt<String> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self.0)
+    }
+}
+
+impl FromStr for MultOpt<String> {
+    type Err = io::Error;
+
+    fn from_str(src: &str) -> Result<Self, Self::Err> {
+        if src == "[]" {
+            Ok(MultOpt::default())
+        } else {
+            let v: Vec<String> = src.split(",").map(|v| v.to_string()).collect();
+            Ok(MultOpt(v))
+        }
+    }
+}

src/main.rs

@@ -0,0 +1,206 @@
+use std::io::{self, Write, BufRead};
+use std::collections::HashMap;
+use termcolor::{BufferWriter, WriteColor, ColorChoice, Color, ColorSpec};
+use anyhow::Result;
+use serde_json::Value;
+use structopt::StructOpt;
+
+mod parser;
+mod cli;
+
+use cli::*;
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "jaxe", about = "A j[son] [pick]axe!")]
+pub(crate) struct Opt {
+    /// Fields to extract, default to extracting all fields
+    #[structopt(short, long, default_value)]
+    extract: MultOpt<String>,
+
+    /// Fields to omit
+    #[structopt(short, long, default_value)]
+    omit: MultOpt<String>,
+
+    /// Do not print non-json lines
+    #[structopt(short = "j", long)]
+    no_omit_json: bool,
+
+    /// Filter by. See parse language
+    #[structopt(short = "f", long)]
+    filter: Vec<String>,
+
+    /// level keys. The first of these keys in the json line will be used as the level of the log line and formatted at the start of the line.
+    #[structopt(short, long)]
+    level: Vec<String>,
+
+    /// Time keys. The first of these keys in the json line will be used as the date of the log line and formatted after the level.
+    #[structopt(short, long)]
+    time: Vec<String>,
+
+    /// Disable colors
+    #[structopt(short, long)]
+    no_colors: bool,
+}
+
+fn level_to_color(level: &str) -> Color {
+    match level {
+        "TRACE" => Color::Magenta,
+        "DEBUG" => Color::Blue,
+        "INFO" => Color::Green,
+        "WARN" => Color::Yellow,
+        "ERROR" => Color::Red,
+        _ => Color::Red
+    }
+}
+
+fn run_filters(filters: &Vec<String>, line: &Value) -> Result<bool> {
+    for filter in filters.iter() {
+        let res = parser::filter(&filter, line)?;
+
+        if ! res {
+            log::debug!("Line ignored, it does not match filter {}", filter);
+            return Ok(false)
+        }
+    }
+
+    Ok(true)
+}
+
+fn write_formatted_line(opts: &Opt, line: Value, output: &termcolor::BufferWriter) -> Result<()> {
+    if ! run_filters(&opts.filter, &line)? {
+        return Ok(())
+    }
+
+    let mut json = serde_json::from_value::<HashMap<String, Value>>(line)?;
+
+    for key in opts.omit.0.iter() {
+        log::debug!("Not writing key {} due to --omit", key);
+        json.remove(key);
+    }
+
+    let mut buffer = output.buffer();
+
+    for key in &opts.level {
+        if let Some(level) = json.get(key).and_then(|s| s.as_str()) {
+            buffer.set_color(ColorSpec::new().set_fg(Some(level_to_color(level))))?;
+            write!(&mut buffer, "{}", level.chars().nth(0).unwrap_or('?'))?;
+            buffer.set_color(ColorSpec::new().set_fg(None))?;
+            write!(&mut buffer, "|")?;
+            json.remove(key);
+
+            break;
+        }
+    }
+
+    for key in &opts.time {
+        if let Some(at) = json.get(key).and_then(|s| s.as_str()) {
+            buffer.set_color(ColorSpec::new().set_fg(None))?;
+            write!(&mut buffer, "{}|", at)?;
+            json.remove(key);
+            break;
+        }
+    }
+
+    let mut keys: Vec<&String> = json.keys().collect();
+    keys.sort();
+
+    for key in keys {
+        if ! opts.extract.0.is_empty() && ! opts.extract.0.contains(key) {
+            log::debug!("Not writing key {} due to --extract", key);
+            continue;
+        }
+
+        let value: &Value = json.get(key).unwrap();
+        buffer.set_color(ColorSpec::new().set_fg(Some(Color::Blue)))?;
+
+        write!(&mut buffer, "{}", key)?;
+
+        if let Some(n) = value.as_str().and_then(|s| s.parse::<u64>().ok()) {
+            buffer.set_color(ColorSpec::new().set_fg(None).set_dimmed(true))?;
+            write!(&mut buffer, "=")?;
+            buffer.set_color(ColorSpec::new().set_fg(Some(Color::Red)).set_dimmed(true))?;
+            write!(&mut buffer, "{} ", n)?;
+        } else if let Some(s) = value.as_str() {
+            buffer.set_color(ColorSpec::new().set_fg(None).set_dimmed(true))?;
+            write!(&mut buffer, "=")?;
+            buffer.set_color(ColorSpec::new().set_fg(None).set_dimmed(false))?;
+            write!(&mut buffer, "{} ", s)?;
+        } else {
+            buffer.set_color(ColorSpec::new().set_fg(None).set_dimmed(true))?;
+            write!(&mut buffer, "=")?;
+            buffer.set_color(ColorSpec::new().set_fg(None))?;
+            write!(&mut buffer, "{} ", value)?;
+        }
+    }
+
+    writeln!(&mut buffer, "")?;
+    output.print(&buffer)?;
+
+    Ok(())
+}
+
+
+fn main() -> io::Result<()> {
+    pretty_env_logger::init();
+
+    let mut opts = Opt::from_args();
+
+    if opts.time.is_empty() {
+        opts.time.push("time".to_owned());
+        opts.time.push("at".to_owned());
+    }
+
+    if opts.level.is_empty() {
+        opts.level.push("level".to_owned());
+    }
+
+    if let Some(e) = std::env::var("JAXE_OMIT").ok()  {
+        opts.omit = MultOpt(e.split(",").map(|s| s.to_owned()).collect());
+    }
+
+    if let Some(e) = std::env::var("JAXE_FILTER").ok()  {
+        opts.filter = vec![e.to_owned()];
+    }
+
+    let mut line_buffer = String::new();
+    let stdin = io::stdin();
+    let mut handle =  stdin.lock();
+
+    let bufwtr = if opts.no_colors {
+        BufferWriter::stdout(ColorChoice::Never)
+    } else {
+        BufferWriter::stdout(ColorChoice::Auto)
+    };
+
+    loop {
+        match handle.read_line(&mut line_buffer) {
+            Err(_) | Ok(0) => {
+                log::debug!("Finished");
+                break;
+            },
+            Ok(c) =>
+                log::debug!("read {} bytes", c)
+        }
+
+        match serde_json::from_str(&line_buffer) {
+            Ok(json) =>
+                write_formatted_line(&opts, json, &bufwtr).unwrap(),
+            Err(err) => {
+                log::debug!("Could not parse line as json: {:?}", err);
+
+                if ! opts.no_omit_json {
+                    let mut obuf = bufwtr.buffer();
+                    obuf.set_color(ColorSpec::new().set_fg(Some(Color::White)).set_dimmed(true))?;
+
+                    write!(&mut obuf, "{}", line_buffer)?;
+
+                    bufwtr.print(&mut obuf)?;
+                }
+            }
+        }
+
+        line_buffer.clear()
+    }
+
+    Ok(())
+}