From 15da09acfd960b107c3750e99944abd1a934ea5a Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sat, 27 Apr 2024 00:41:56 +0200 Subject: [PATCH 1/6] Working raw --- src/jq.rs | 44 ++++++++++++++++++++++++---------- src/lib.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 14 deletions(-) diff --git a/src/jq.rs b/src/jq.rs index 7e0e114..22085ab 100644 --- a/src/jq.rs +++ b/src/jq.rs @@ -87,14 +87,20 @@ impl Jq { /// Run the jq program against an input. pub fn execute(&mut self, input: CString) -> Result { let mut parser = Parser::new(); - self.process(parser.parse(input)?) + self.process(parser.parse(input)?, false) + } + + /// Run the jq program against an input, while returning raw output. + pub fn execute_raw(&mut self, input: CString) -> Result { + let mut parser = Parser::new(); + self.process(parser.parse(input)?, true) } /// Unwind the parser and return the rendered result. /// /// When this results in `Err`, the String value should contain a message about /// what failed. - fn process(&mut self, initial_value: JV) -> Result { + fn process(&mut self, initial_value: JV, use_raw_output: bool) -> Result { let mut buf = String::new(); unsafe { @@ -106,7 +112,7 @@ impl Jq { // it is no longer needed. drop(initial_value); - dump(self, &mut buf)?; + dump(self, &mut buf, use_raw_output)?; } Ok(buf) @@ -129,7 +135,15 @@ impl JV { let dump = JV { ptr: unsafe { jv_dump_string(jv_copy(self.ptr), 0) }, }; - unsafe { get_string_value(jv_string_value(dump.ptr)) } + dump.as_string() + } + + /// Convert the current `JV` into the raw rendering of itself. + pub fn as_raw_string(&self) -> Result { + let copied = JV { + ptr: unsafe { jv_copy(self.ptr) }, + }; + copied.as_string() } /// Attempts to extract feedback from jq if the JV is invalid. @@ -168,15 +182,17 @@ impl JV { } pub fn as_string(&self) -> Result { - unsafe { - if jv_get_kind(self.ptr) == jv_kind_JV_KIND_STRING { - get_string_value(jv_string_value(self.ptr)) - } else { - Err(Error::Unknown) - } + if self.is_string() { + unsafe { get_string_value(jv_string_value(self.ptr)) } + } else { + Err(Error::Unknown) } } + pub fn is_string(&self) -> bool { + unsafe { jv_get_kind(self.ptr) == jv_kind_JV_KIND_STRING } + } + pub fn is_valid(&self) -> bool { unsafe { jv_get_kind(self.ptr) != jv_kind_JV_KIND_INVALID } } @@ -256,7 +272,7 @@ unsafe fn get_string_value(value: *const c_char) -> Result { } /// Renders the data from the parser and pushes it into the buffer. -unsafe fn dump(jq: &Jq, buf: &mut String) -> Result<()> { +unsafe fn dump(jq: &Jq, buf: &mut String, use_raw_output: bool) -> Result<()> { // Looks a lot like an iterator... let mut value = JV { @@ -264,7 +280,11 @@ unsafe fn dump(jq: &Jq, buf: &mut String) -> Result<()> { }; while value.is_valid() { - let s = value.as_dump_string()?; + let s: String = if use_raw_output && value.is_string() { + value.as_raw_string()? + } else { + value.as_dump_string()? + }; buf.push_str(&s); buf.push('\n'); diff --git a/src/lib.rs b/src/lib.rs index 1f4c90f..302a835 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -165,6 +165,16 @@ pub fn run(program: &str, data: &str) -> Result { compile(program)?.run(data) } +/// Run a jq program on a blob of json data and return raw output. +/// +/// In the case of failure to run the program, feedback from the jq api will be +/// available in the supplied `String` value. +/// Failures can occur for a variety of reasons, but mostly you'll see them as +/// a result of bad jq program syntax, or invalid json data. +pub fn run_raw(program: &str, data: &str) -> Result { + compile(program)?.run_raw(data) +} + /// A pre-compiled jq program which can be run against different inputs. pub struct JqProgram { jq: jq::Jq, @@ -173,6 +183,15 @@ pub struct JqProgram { impl JqProgram { /// Runs a json string input against a pre-compiled jq program. pub fn run(&mut self, data: &str) -> Result { + self.execute(data, false) + } + + /// Runs a json string input against a pre-compiled jq program and returns raw output + pub fn run_raw(&mut self, data: &str) -> Result { + self.execute(data, true) + } + + fn execute(&mut self, data: &str, use_raw_output: bool) -> Result { if data.trim().is_empty() { // During work on #4, #7, the parser test which allows us to avoid a memory // error shows that an empty input just yields an empty response BUT our @@ -180,7 +199,10 @@ impl JqProgram { return Ok("".into()); } let input = CString::new(data)?; - self.jq.execute(input) + match use_raw_output { + true => self.jq.execute_raw(input), + false => self.jq.execute(input), + } } } @@ -195,7 +217,7 @@ pub fn compile(program: &str) -> Result { #[cfg(test)] mod test { - use super::{compile, run, Error}; + use super::{compile, run, run_raw, Error}; use matches::assert_matches; use serde_json; @@ -304,6 +326,50 @@ mod test { assert_matches!(res, Err(Error::System { .. })); } + #[test] + fn run_raw_string_nested_test() { + let res = run_raw( + r#"{name, test: "\(.)"}"#, + r#"{"name":"john"}"#, + ); + let expected = r#"{"name":"john","test":"{\"name\":\"john\"}"} +"#; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn run_raw_string_test() { + let res = run_raw( + r#""\(.)""#, + r#"{"name":"john"}"#, + ); + let expected = r#"{"name":"john"} +"#; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn run_raw_object_test() { + let res = run_raw( + r#"."#, + r#"{"name":"john"}"#, + ); + let expected = r#"{"name":"john"} +"#; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn run_test() { + let res = run( + r#""\(.)""#, + r#"{"name":"john"}"#, + ); + let expected = r#""{\"name\":\"john\"}" +"#; + assert_eq!(res.unwrap(), expected); + } + pub mod mem_errors { //! Attempting run a program resulting in bad field access has been //! shown to sometimes trigger a use after free or double free memory From e17d9be7b90e48037a9a89839386803f8cac1451 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sat, 27 Apr 2024 02:49:58 +0200 Subject: [PATCH 2/6] Colorization and indentation --- src/jq.rs | 84 +++++++++++++++++++++++++++++------- src/lib.rs | 113 ++++++++++++++++++++++++++++++++++++++++--------- src/options.rs | 112 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 274 insertions(+), 35 deletions(-) create mode 100644 src/options.rs diff --git a/src/jq.rs b/src/jq.rs index 22085ab..881689b 100644 --- a/src/jq.rs +++ b/src/jq.rs @@ -4,12 +4,15 @@ //! These are building blocks and not intended for use from the public API. use crate::errors::{Error, Result}; +use crate::options::{JqColorization, JqIndentation}; +use crate::JqOptions; use jq_sys::{ - jq_compile, jq_format_error, jq_get_exit_code, jq_halted, jq_init, jq_next, jq_set_error_cb, - jq_start, jq_state, jq_teardown, jv, jv_copy, jv_dump_string, jv_free, jv_get_kind, - jv_invalid_get_msg, jv_invalid_has_msg, jv_kind_JV_KIND_INVALID, jv_kind_JV_KIND_NUMBER, - jv_kind_JV_KIND_STRING, jv_number_value, jv_parser, jv_parser_free, jv_parser_new, - jv_parser_next, jv_parser_set_buf, jv_string_value, + jq_compile, jq_format_error, jq_get_exit_code, jq_halted, jq_init, jq_next, jq_set_colors, + jq_set_error_cb, jq_start, jq_state, jq_teardown, jv, jv_copy, jv_dump_string, jv_free, + jv_get_kind, jv_invalid_get_msg, jv_invalid_has_msg, jv_kind_JV_KIND_INVALID, + jv_kind_JV_KIND_NUMBER, jv_kind_JV_KIND_STRING, jv_number_value, jv_parser, jv_parser_free, + jv_parser_new, jv_parser_next, jv_parser_set_buf, jv_print_flags_JV_PRINT_COLOR, + jv_print_flags_JV_PRINT_PRETTY, jv_print_flags_JV_PRINT_TAB, jv_string_value, }; use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_void}; @@ -84,23 +87,34 @@ impl Jq { } } + pub unsafe fn set_colors(&self, colors: &str) -> Result<()> { + let c_colors = CString::new(colors)?; + let val = jq_set_colors(c_colors.as_ptr()); + + if val == 1 { + return Ok(()); + } + + Err(Error::Unknown) + } + /// Run the jq program against an input. pub fn execute(&mut self, input: CString) -> Result { let mut parser = Parser::new(); - self.process(parser.parse(input)?, false) + self.process(parser.parse(input)?, JqOptions::default()) } /// Run the jq program against an input, while returning raw output. - pub fn execute_raw(&mut self, input: CString) -> Result { + pub fn execute_advanced(&mut self, input: CString, options: JqOptions) -> Result { let mut parser = Parser::new(); - self.process(parser.parse(input)?, true) + self.process(parser.parse(input)?, options) } /// Unwind the parser and return the rendered result. /// /// When this results in `Err`, the String value should contain a message about /// what failed. - fn process(&mut self, initial_value: JV, use_raw_output: bool) -> Result { + fn process(&mut self, initial_value: JV, options: JqOptions) -> Result { let mut buf = String::new(); unsafe { @@ -112,7 +126,7 @@ impl Jq { // it is no longer needed. drop(initial_value); - dump(self, &mut buf, use_raw_output)?; + dump(self, &mut buf, options)?; } Ok(buf) @@ -129,11 +143,23 @@ struct JV { ptr: jv, } +fn jv_print_indent_flags(n: u32) -> u32 { + if n > 7 { + return jv_print_flags_JV_PRINT_TAB | jv_print_flags_JV_PRINT_PRETTY; + } + + if n == 0 { + return 0; + } + + n << 8 | jv_print_flags_JV_PRINT_PRETTY +} + impl JV { /// Convert the current `JV` into the "dump string" rendering of itself. - pub fn as_dump_string(&self) -> Result { + pub fn as_dump_string(&self, flags: u32) -> Result { let dump = JV { - ptr: unsafe { jv_dump_string(jv_copy(self.ptr), 0) }, + ptr: unsafe { jv_dump_string(jv_copy(self.ptr), flags as i32) }, }; dump.as_string() } @@ -272,18 +298,46 @@ unsafe fn get_string_value(value: *const c_char) -> Result { } /// Renders the data from the parser and pushes it into the buffer. -unsafe fn dump(jq: &Jq, buf: &mut String, use_raw_output: bool) -> Result<()> { +unsafe fn dump(jq: &Jq, buf: &mut String, options: JqOptions) -> Result<()> { // Looks a lot like an iterator... let mut value = JV { ptr: jq_next(jq.state), }; + let mut dumpoptions = jv_print_indent_flags(2); + match options.indentation { + JqIndentation::Compact => { + dumpoptions &= !(jv_print_flags_JV_PRINT_TAB | jv_print_indent_flags(7)); + } + JqIndentation::Tabs => { + dumpoptions &= !jv_print_indent_flags(7); + dumpoptions |= jv_print_flags_JV_PRINT_TAB | jv_print_flags_JV_PRINT_PRETTY; + } + JqIndentation::Spaces(indent) => { + dumpoptions &= !(jv_print_flags_JV_PRINT_TAB | jv_print_indent_flags(7)); + dumpoptions |= jv_print_indent_flags(indent as u32); + } + }; + + match options.colorization { + JqColorization::Custom(colors) => { + jq.set_colors(colors)?; + dumpoptions |= jv_print_flags_JV_PRINT_COLOR; + } + JqColorization::Colorize => { + dumpoptions |= jv_print_flags_JV_PRINT_COLOR; + } + JqColorization::Monochrome => { + dumpoptions &= !jv_print_flags_JV_PRINT_COLOR; + } + } + while value.is_valid() { - let s: String = if use_raw_output && value.is_string() { + let s: String = if options.raw_output && value.is_string() { value.as_raw_string()? } else { - value.as_dump_string()? + value.as_dump_string(dumpoptions)? }; buf.push_str(&s); buf.push('\n'); diff --git a/src/lib.rs b/src/lib.rs index 302a835..ae48a99 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -150,10 +150,12 @@ extern crate serde_json; mod errors; mod jq; +mod options; use std::ffi::CString; pub use errors::{Error, Result}; +use options::JqOptions; /// Run a jq program on a blob of json data. /// @@ -171,8 +173,8 @@ pub fn run(program: &str, data: &str) -> Result { /// available in the supplied `String` value. /// Failures can occur for a variety of reasons, but mostly you'll see them as /// a result of bad jq program syntax, or invalid json data. -pub fn run_raw(program: &str, data: &str) -> Result { - compile(program)?.run_raw(data) +pub fn run_advanced(program: &str, data: &str, options: JqOptions) -> Result { + compile(program)?.run_advanced(data, options) } /// A pre-compiled jq program which can be run against different inputs. @@ -183,15 +185,15 @@ pub struct JqProgram { impl JqProgram { /// Runs a json string input against a pre-compiled jq program. pub fn run(&mut self, data: &str) -> Result { - self.execute(data, false) + self.execute(data, JqOptions::default()) } - /// Runs a json string input against a pre-compiled jq program and returns raw output - pub fn run_raw(&mut self, data: &str) -> Result { - self.execute(data, true) + /// Runs a json string input against a pre-compiled jq program with additional jq options + pub fn run_advanced(&mut self, data: &str, options: JqOptions) -> Result { + self.execute(data, options) } - fn execute(&mut self, data: &str, use_raw_output: bool) -> Result { + fn execute(&mut self, data: &str, options: JqOptions) -> Result { if data.trim().is_empty() { // During work on #4, #7, the parser test which allows us to avoid a memory // error shows that an empty input just yields an empty response BUT our @@ -199,10 +201,7 @@ impl JqProgram { return Ok("".into()); } let input = CString::new(data)?; - match use_raw_output { - true => self.jq.execute_raw(input), - false => self.jq.execute(input), - } + self.jq.execute_advanced(input, options) } } @@ -217,7 +216,12 @@ pub fn compile(program: &str) -> Result { #[cfg(test)] mod test { - use super::{compile, run, run_raw, Error}; + use crate::{ + options::{JqColorization, JqIndentation}, + JqOptions, + }; + + use super::{compile, run, run_advanced, Error}; use matches::assert_matches; use serde_json; @@ -327,10 +331,11 @@ mod test { } #[test] - fn run_raw_string_nested_test() { - let res = run_raw( + fn run_raw_string_nested() { + let res = run_advanced( r#"{name, test: "\(.)"}"#, r#"{"name":"john"}"#, + JqOptions::default().with_raw_output(true), ); let expected = r#"{"name":"john","test":"{\"name\":\"john\"}"} "#; @@ -338,10 +343,11 @@ mod test { } #[test] - fn run_raw_string_test() { - let res = run_raw( + fn run_raw_string() { + let res = run_advanced( r#""\(.)""#, r#"{"name":"john"}"#, + JqOptions::default().with_raw_output(true), ); let expected = r#"{"name":"john"} "#; @@ -350,9 +356,10 @@ mod test { #[test] fn run_raw_object_test() { - let res = run_raw( + let res = run_advanced( r#"."#, r#"{"name":"john"}"#, + JqOptions::default().with_raw_output(true), ); let expected = r#"{"name":"john"} "#; @@ -360,11 +367,77 @@ mod test { } #[test] - fn run_test() { - let res = run( - r#""\(.)""#, + fn run_color_test() { + let res = run_advanced( + r#"."#, + r#"{"name":"john"}"#, + JqOptions::default().with_colorization(JqColorization::Colorize), + ); + let expected = "\u{1b}[1;39m{\u{1b}[0m\u{1b}[34;1m\"name\"\u{1b}[0m\u{1b}[1;39m:\u{1b}[0m\u{1b}[0;32m\"john\"\u{1b}[0m\u{1b}[1;39m\u{1b}[1;39m}\u{1b}[0m\n"; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn run_custom_color_test() { + let res = run_advanced( + r#"."#, + r#"{"name":"john"}"#, + JqOptions::default().with_colorization(JqColorization::Custom("0;34:0;34:0;34:0;34:0;34:0;34:0;34:0;34")), + ); + let expected = "\u{1b}[0;34m{\u{1b}[0m\u{1b}[34;1m\"name\"\u{1b}[0m\u{1b}[0;34m:\u{1b}[0m\u{1b}[0;34m\"john\"\u{1b}[0m\u{1b}[0;34m\u{1b}[0;34m}\u{1b}[0m\n"; + assert_eq!(res.unwrap(), expected); + } + #[test] + fn run_monochrome_test() { + let res = run_advanced( + r#"."#, + r#"{"name":"john"}"#, + JqOptions::default().with_colorization(JqColorization::Monochrome), + ); + let expected = r#"{"name":"john"} +"#; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn run_indent_4() { + let res = run_advanced( + r#"."#, + r#"{"name":"john"}"#, + JqOptions::default().with_indentation(JqIndentation::Spaces(4)), + ); + let expected = r#"{ + "name": "john" +} +"#; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn run_indent_minus1() { + let res = run_advanced( + r#"."#, r#"{"name":"john"}"#, + JqOptions::default().with_indentation(JqIndentation::Spaces(-1)), ); + let expected = "{\n\t\"name\": \"john\"\n}\n"; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn run_indent_tab() { + let res = run_advanced( + r#"."#, + r#"{"name":"john"}"#, + JqOptions::default().with_indentation(JqIndentation::Tabs), + ); + let expected = "{\n\t\"name\": \"john\"\n}\n"; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn run_test() { + let res = run(r#""\(.)""#, r#"{"name":"john"}"#); let expected = r#""{\"name\":\"john\"}" "#; assert_eq!(res.unwrap(), expected); diff --git a/src/options.rs b/src/options.rs new file mode 100644 index 0000000..1ccc4b7 --- /dev/null +++ b/src/options.rs @@ -0,0 +1,112 @@ +/// Struct containing options that can be passed to jq to customize input/output +pub struct JqOptions<'a> { + pub raw_output: bool, + pub raw_input: bool, + pub slurp: bool, + pub sort_keys: bool, + pub colorization: JqColorization<'a>, + pub indentation: JqIndentation, +} + +impl<'a> Default for JqOptions<'a> { + fn default() -> Self { + JqOptions { + raw_input: false, // TODO + raw_output: false, + slurp: false, // TODO + sort_keys: false, // TODO + indentation: JqIndentation::Compact, + colorization: JqColorization::Monochrome, + } + } +} + +impl<'a> JqOptions<'a> { + pub fn with_raw_output(&self, raw_output: bool) -> Self { + JqOptions { + raw_output, + raw_input: self.raw_input, + slurp: self.slurp, + sort_keys: self.sort_keys, + colorization: self.colorization, + indentation: self.indentation, + } + } + + pub fn with_raw_input(&self, raw_input: bool) -> Self { + JqOptions { + raw_output: self.raw_output, + raw_input, + slurp: self.slurp, + sort_keys: self.sort_keys, + colorization: self.colorization, + indentation: self.indentation, + } + } + + pub fn with_slurp(&self, slurp: bool) -> Self { + JqOptions { + raw_output: self.raw_output, + raw_input: self.raw_input, + slurp, + sort_keys: self.sort_keys, + colorization: self.colorization, + indentation: self.indentation, + } + } + + pub fn with_sort_keys(&self, sort_keys: bool) -> Self { + JqOptions { + raw_output: self.raw_output, + raw_input: self.raw_input, + slurp: self.slurp, + sort_keys, + colorization: self.colorization, + indentation: self.indentation, + } + } + + pub fn with_colorization(&self, colorization: JqColorization<'a>) -> Self { + JqOptions { + raw_output: self.raw_output, + raw_input: self.raw_input, + slurp: self.slurp, + sort_keys: self.sort_keys, + colorization, + indentation: self.indentation, + } + } + + pub fn with_indentation(&self, indentation: JqIndentation) -> Self { + JqOptions { + raw_output: self.raw_output, + raw_input: self.raw_input, + slurp: self.slurp, + sort_keys: self.sort_keys, + colorization: self.colorization, + indentation, + } + } +} + +/// The indentation options for jq +#[derive(Copy, Clone)] +pub enum JqIndentation { + /// Don't indent, fully compact + Compact, + /// Use tabs for indentation + Tabs, + /// Use spaces for indentation + Spaces(i32), +} + +/// The two possible colorization options +#[derive(Copy, Clone)] +pub enum JqColorization<'a> { + /// Apply custom colors + Custom(&'a str), + /// Apply full colorization + Colorize, + /// Don't apply colorization + Monochrome, +} From 7a30094dff727af9fcd84a9eecc18a00ba048c73 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sun, 28 Apr 2024 03:01:22 +0200 Subject: [PATCH 3/6] Fixes --- src/jq.rs | 48 +++++++++++++++++++++++++------------------- src/lib.rs | 54 +++++++++++++++++++++++++++++++++++--------------- src/options.rs | 18 ----------------- 3 files changed, 66 insertions(+), 54 deletions(-) diff --git a/src/jq.rs b/src/jq.rs index 881689b..206e721 100644 --- a/src/jq.rs +++ b/src/jq.rs @@ -12,10 +12,12 @@ use jq_sys::{ jv_get_kind, jv_invalid_get_msg, jv_invalid_has_msg, jv_kind_JV_KIND_INVALID, jv_kind_JV_KIND_NUMBER, jv_kind_JV_KIND_STRING, jv_number_value, jv_parser, jv_parser_free, jv_parser_new, jv_parser_next, jv_parser_set_buf, jv_print_flags_JV_PRINT_COLOR, - jv_print_flags_JV_PRINT_PRETTY, jv_print_flags_JV_PRINT_TAB, jv_string_value, + jv_print_flags_JV_PRINT_PRETTY, jv_print_flags_JV_PRINT_SORTED, jv_print_flags_JV_PRINT_TAB, + jv_string_value, }; use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_void}; +use std::sync::{Mutex, MutexGuard}; pub struct Jq { state: *mut jq_state, @@ -87,24 +89,7 @@ impl Jq { } } - pub unsafe fn set_colors(&self, colors: &str) -> Result<()> { - let c_colors = CString::new(colors)?; - let val = jq_set_colors(c_colors.as_ptr()); - - if val == 1 { - return Ok(()); - } - - Err(Error::Unknown) - } - - /// Run the jq program against an input. - pub fn execute(&mut self, input: CString) -> Result { - let mut parser = Parser::new(); - self.process(parser.parse(input)?, JqOptions::default()) - } - - /// Run the jq program against an input, while returning raw output. + /// Run the jq program against an input, using options. pub fn execute_advanced(&mut self, input: CString, options: JqOptions) -> Result { let mut parser = Parser::new(); self.process(parser.parse(input)?, options) @@ -118,6 +103,9 @@ impl Jq { let mut buf = String::new(); unsafe { + static LOCK: Mutex<()> = Mutex::new(()); + let _guard: MutexGuard<()> = LOCK.lock().unwrap(); + // `jq_start` seems to be a consuming call. // In order to avoid a double-free, when `initial_value` is dropped, // we have to use `jv_copy` on the inner `jv`. @@ -320,12 +308,15 @@ unsafe fn dump(jq: &Jq, buf: &mut String, options: JqOptions) -> Result<()> { } }; + // Reset colors + try_set_colors("")?; match options.colorization { JqColorization::Custom(colors) => { - jq.set_colors(colors)?; + try_set_colors(colors)?; dumpoptions |= jv_print_flags_JV_PRINT_COLOR; } JqColorization::Colorize => { + // This is the default, but we need to restore it as it might have been overwritten by a previous run. dumpoptions |= jv_print_flags_JV_PRINT_COLOR; } JqColorization::Monochrome => { @@ -333,6 +324,10 @@ unsafe fn dump(jq: &Jq, buf: &mut String, options: JqOptions) -> Result<()> { } } + if options.sort_keys { + dumpoptions |= jv_print_flags_JV_PRINT_SORTED; + } + while value.is_valid() { let s: String = if options.raw_output && value.is_string() { value.as_raw_string()? @@ -376,6 +371,19 @@ unsafe fn dump(jq: &Jq, buf: &mut String, options: JqOptions) -> Result<()> { } } +unsafe fn try_set_colors(colors: &str) -> Result<()> { + let c_colors = CString::new(colors)?; + let val = jq_set_colors(c_colors.as_ptr()); + + if val != 1 { + return Err(Error::System { + reason: Some("Error while setting colors".to_string()), + }); + } + + Ok(()) +} + /// Various exit codes jq checks for during the `if (jq_halted(jq))` branch of /// their processing loop. /// diff --git a/src/lib.rs b/src/lib.rs index ae48a99..e78d4ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -223,12 +223,11 @@ mod test { use super::{compile, run, run_advanced, Error}; use matches::assert_matches; - use serde_json; #[test] fn reuse_compiled_program() { let query = r#"if . == 0 then "zero" elif . == 1 then "one" else "many" end"#; - let mut prog = compile(&query).unwrap(); + let mut prog = compile(query).unwrap(); assert_eq!(prog.run("2").unwrap(), "\"many\"\n"); assert_eq!(prog.run("1").unwrap(), "\"one\"\n"); assert_eq!(prog.run("0").unwrap(), "\"zero\"\n"); @@ -242,8 +241,8 @@ mod test { // Basically this test is just to check that the state pointers returned by // `jq::init()` are completely independent and don't share any global state. - let mut prog1 = compile(&query1).unwrap(); - let mut prog2 = compile(&query2).unwrap(); + let mut prog1 = compile(query1).unwrap(); + let mut prog2 = compile(query2).unwrap(); assert_eq!(prog1.run(input).unwrap(), "\"foo\"\n"); assert_eq!(prog2.run(input).unwrap(), "123\n"); @@ -331,7 +330,7 @@ mod test { } #[test] - fn run_raw_string_nested() { + fn raw_output_string_nested() { let res = run_advanced( r#"{name, test: "\(.)"}"#, r#"{"name":"john"}"#, @@ -343,7 +342,7 @@ mod test { } #[test] - fn run_raw_string() { + fn raw_output_string() { let res = run_advanced( r#""\(.)""#, r#"{"name":"john"}"#, @@ -355,7 +354,7 @@ mod test { } #[test] - fn run_raw_object_test() { + fn raw_output_object() { let res = run_advanced( r#"."#, r#"{"name":"john"}"#, @@ -367,28 +366,40 @@ mod test { } #[test] - fn run_color_test() { + fn colorize() { let res = run_advanced( r#"."#, r#"{"name":"john"}"#, JqOptions::default().with_colorization(JqColorization::Colorize), ); - let expected = "\u{1b}[1;39m{\u{1b}[0m\u{1b}[34;1m\"name\"\u{1b}[0m\u{1b}[1;39m:\u{1b}[0m\u{1b}[0;32m\"john\"\u{1b}[0m\u{1b}[1;39m\u{1b}[1;39m}\u{1b}[0m\n"; + let expected = "\u{1b}[1;39m{\u{1b}[0m\u{1b}[1;34m\"name\"\u{1b}[0m\u{1b}[1;39m:\u{1b}[0m\u{1b}[0;32m\"john\"\u{1b}[0m\u{1b}[1;39m}\u{1b}[0m\n"; assert_eq!(res.unwrap(), expected); } #[test] - fn run_custom_color_test() { + fn custom_colorize_twice() { let res = run_advanced( r#"."#, r#"{"name":"john"}"#, JqOptions::default().with_colorization(JqColorization::Custom("0;34:0;34:0;34:0;34:0;34:0;34:0;34:0;34")), ); - let expected = "\u{1b}[0;34m{\u{1b}[0m\u{1b}[34;1m\"name\"\u{1b}[0m\u{1b}[0;34m:\u{1b}[0m\u{1b}[0;34m\"john\"\u{1b}[0m\u{1b}[0;34m\u{1b}[0;34m}\u{1b}[0m\n"; + let expected = "\u{1b}[0;34m{\u{1b}[0m\u{1b}[0;34m\"name\"\u{1b}[0m\u{1b}[0;34m:\u{1b}[0m\u{1b}[0;34m\"john\"\u{1b}[0m\u{1b}[0;34m}\u{1b}[0m\n"; assert_eq!(res.unwrap(), expected); } + #[test] - fn run_monochrome_test() { + fn custom_colorize() { + let res = run_advanced( + r#"."#, + r#"{"name":"john"}"#, + JqOptions::default().with_colorization(JqColorization::Custom("0;34:0;34:0;34:0;34:0;34:0;34:0;34:0;34")), + ); + let expected = "\u{1b}[0;34m{\u{1b}[0m\u{1b}[0;34m\"name\"\u{1b}[0m\u{1b}[0;34m:\u{1b}[0m\u{1b}[0;34m\"john\"\u{1b}[0m\u{1b}[0;34m}\u{1b}[0m\n"; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn monochrome() { let res = run_advanced( r#"."#, r#"{"name":"john"}"#, @@ -400,7 +411,7 @@ mod test { } #[test] - fn run_indent_4() { + fn indent_4_spaces() { let res = run_advanced( r#"."#, r#"{"name":"john"}"#, @@ -414,7 +425,7 @@ mod test { } #[test] - fn run_indent_minus1() { + fn indent_minus1_spaces() { let res = run_advanced( r#"."#, r#"{"name":"john"}"#, @@ -425,7 +436,7 @@ mod test { } #[test] - fn run_indent_tab() { + fn indent_tabs() { let res = run_advanced( r#"."#, r#"{"name":"john"}"#, @@ -436,7 +447,18 @@ mod test { } #[test] - fn run_test() { + fn sort_keys() { + let res = run_advanced( + r#"."#, + r#"{"c":"fourth","b":{"c":"third","a": "second"},"a":"first"}"#, + JqOptions::default().with_sort_keys(true), + ); + let expected = "{\"a\":\"first\",\"b\":{\"a\":\"second\",\"c\":\"third\"},\"c\":\"fourth\"}\n"; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn basic() { let res = run(r#""\(.)""#, r#"{"name":"john"}"#); let expected = r#""{\"name\":\"john\"}" "#; diff --git a/src/options.rs b/src/options.rs index 1ccc4b7..050923a 100644 --- a/src/options.rs +++ b/src/options.rs @@ -2,7 +2,6 @@ pub struct JqOptions<'a> { pub raw_output: bool, pub raw_input: bool, - pub slurp: bool, pub sort_keys: bool, pub colorization: JqColorization<'a>, pub indentation: JqIndentation, @@ -13,7 +12,6 @@ impl<'a> Default for JqOptions<'a> { JqOptions { raw_input: false, // TODO raw_output: false, - slurp: false, // TODO sort_keys: false, // TODO indentation: JqIndentation::Compact, colorization: JqColorization::Monochrome, @@ -26,7 +24,6 @@ impl<'a> JqOptions<'a> { JqOptions { raw_output, raw_input: self.raw_input, - slurp: self.slurp, sort_keys: self.sort_keys, colorization: self.colorization, indentation: self.indentation, @@ -37,18 +34,6 @@ impl<'a> JqOptions<'a> { JqOptions { raw_output: self.raw_output, raw_input, - slurp: self.slurp, - sort_keys: self.sort_keys, - colorization: self.colorization, - indentation: self.indentation, - } - } - - pub fn with_slurp(&self, slurp: bool) -> Self { - JqOptions { - raw_output: self.raw_output, - raw_input: self.raw_input, - slurp, sort_keys: self.sort_keys, colorization: self.colorization, indentation: self.indentation, @@ -59,7 +44,6 @@ impl<'a> JqOptions<'a> { JqOptions { raw_output: self.raw_output, raw_input: self.raw_input, - slurp: self.slurp, sort_keys, colorization: self.colorization, indentation: self.indentation, @@ -70,7 +54,6 @@ impl<'a> JqOptions<'a> { JqOptions { raw_output: self.raw_output, raw_input: self.raw_input, - slurp: self.slurp, sort_keys: self.sort_keys, colorization, indentation: self.indentation, @@ -81,7 +64,6 @@ impl<'a> JqOptions<'a> { JqOptions { raw_output: self.raw_output, raw_input: self.raw_input, - slurp: self.slurp, sort_keys: self.sort_keys, colorization: self.colorization, indentation, From 1c01fee79da754c83b083602dc519fe5d352e8d0 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sun, 28 Apr 2024 03:09:25 +0200 Subject: [PATCH 4/6] Provide comments and expose options --- src/lib.rs | 2 +- src/options.rs | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e78d4ac..415b46c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -155,7 +155,7 @@ mod options; use std::ffi::CString; pub use errors::{Error, Result}; -use options::JqOptions; +pub use options::JqOptions; /// Run a jq program on a blob of json data. /// diff --git a/src/options.rs b/src/options.rs index 050923a..c9b5f34 100644 --- a/src/options.rs +++ b/src/options.rs @@ -1,9 +1,14 @@ /// Struct containing options that can be passed to jq to customize input/output pub struct JqOptions<'a> { + /// Output raw strings instead of quoted and escaped pub raw_output: bool, + /// Interpret each input as string instead of json pub raw_input: bool, + /// Order the keys pub sort_keys: bool, + /// Use colors for the output pub colorization: JqColorization<'a>, + /// Apply indentation for the output pub indentation: JqIndentation, } @@ -12,7 +17,7 @@ impl<'a> Default for JqOptions<'a> { JqOptions { raw_input: false, // TODO raw_output: false, - sort_keys: false, // TODO + sort_keys: false, indentation: JqIndentation::Compact, colorization: JqColorization::Monochrome, } @@ -20,6 +25,7 @@ impl<'a> Default for JqOptions<'a> { } impl<'a> JqOptions<'a> { + /// Output raw strings instead of quoted and escaped pub fn with_raw_output(&self, raw_output: bool) -> Self { JqOptions { raw_output, @@ -30,6 +36,7 @@ impl<'a> JqOptions<'a> { } } + /// Interpret each input as string instead of json pub fn with_raw_input(&self, raw_input: bool) -> Self { JqOptions { raw_output: self.raw_output, @@ -40,6 +47,7 @@ impl<'a> JqOptions<'a> { } } + /// Order the keys pub fn with_sort_keys(&self, sort_keys: bool) -> Self { JqOptions { raw_output: self.raw_output, @@ -50,6 +58,7 @@ impl<'a> JqOptions<'a> { } } + /// Use colors for the output pub fn with_colorization(&self, colorization: JqColorization<'a>) -> Self { JqOptions { raw_output: self.raw_output, @@ -60,6 +69,7 @@ impl<'a> JqOptions<'a> { } } + /// Apply indentation for the output pub fn with_indentation(&self, indentation: JqIndentation) -> Self { JqOptions { raw_output: self.raw_output, From da6085bd7d3a2579db71de2544e144904c397cff Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sun, 28 Apr 2024 03:28:04 +0200 Subject: [PATCH 5/6] Support raw input --- src/jq.rs | 18 +++++++++++------- src/lib.rs | 25 +++++++++++++++++++++---- src/options.rs | 2 +- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/jq.rs b/src/jq.rs index 206e721..1d90f8f 100644 --- a/src/jq.rs +++ b/src/jq.rs @@ -13,7 +13,7 @@ use jq_sys::{ jv_kind_JV_KIND_NUMBER, jv_kind_JV_KIND_STRING, jv_number_value, jv_parser, jv_parser_free, jv_parser_new, jv_parser_next, jv_parser_set_buf, jv_print_flags_JV_PRINT_COLOR, jv_print_flags_JV_PRINT_PRETTY, jv_print_flags_JV_PRINT_SORTED, jv_print_flags_JV_PRINT_TAB, - jv_string_value, + jv_string, jv_string_value, }; use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_void}; @@ -90,16 +90,16 @@ impl Jq { } /// Run the jq program against an input, using options. - pub fn execute_advanced(&mut self, input: CString, options: JqOptions) -> Result { + pub fn execute_advanced(&mut self, input: CString, options: &JqOptions) -> Result { let mut parser = Parser::new(); - self.process(parser.parse(input)?, options) + self.process(parser.parse(input, options)?, options) } /// Unwind the parser and return the rendered result. /// /// When this results in `Err`, the String value should contain a message about /// what failed. - fn process(&mut self, initial_value: JV, options: JqOptions) -> Result { + fn process(&mut self, initial_value: JV, options: &JqOptions) -> Result { let mut buf = String::new(); unsafe { @@ -233,7 +233,7 @@ impl Parser { } } - pub fn parse(&mut self, input: CString) -> Result { + pub fn parse(&mut self, input: CString, options: &JqOptions) -> Result { // For a single run, we could set this to `1` (aka `true`) but this will // break the repeated `JqProgram` usage. // It may be worth exposing this to the caller so they can set it for each @@ -255,7 +255,11 @@ impl Parser { }; let value = JV { - ptr: unsafe { jv_parser_next(self.ptr) }, + ptr: if !options.raw_input { + unsafe { jv_parser_next(self.ptr) } + } else { + unsafe { jv_string(input.as_ptr()) } + }, }; if value.is_valid() { Ok(value) @@ -286,7 +290,7 @@ unsafe fn get_string_value(value: *const c_char) -> Result { } /// Renders the data from the parser and pushes it into the buffer. -unsafe fn dump(jq: &Jq, buf: &mut String, options: JqOptions) -> Result<()> { +unsafe fn dump(jq: &Jq, buf: &mut String, options: &JqOptions) -> Result<()> { // Looks a lot like an iterator... let mut value = JV { diff --git a/src/lib.rs b/src/lib.rs index 415b46c..4a98c6c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -201,7 +201,7 @@ impl JqProgram { return Ok("".into()); } let input = CString::new(data)?; - self.jq.execute_advanced(input, options) + self.jq.execute_advanced(input, &options) } } @@ -381,7 +381,9 @@ mod test { let res = run_advanced( r#"."#, r#"{"name":"john"}"#, - JqOptions::default().with_colorization(JqColorization::Custom("0;34:0;34:0;34:0;34:0;34:0;34:0;34:0;34")), + JqOptions::default().with_colorization(JqColorization::Custom( + "0;34:0;34:0;34:0;34:0;34:0;34:0;34:0;34", + )), ); let expected = "\u{1b}[0;34m{\u{1b}[0m\u{1b}[0;34m\"name\"\u{1b}[0m\u{1b}[0;34m:\u{1b}[0m\u{1b}[0;34m\"john\"\u{1b}[0m\u{1b}[0;34m}\u{1b}[0m\n"; assert_eq!(res.unwrap(), expected); @@ -392,7 +394,9 @@ mod test { let res = run_advanced( r#"."#, r#"{"name":"john"}"#, - JqOptions::default().with_colorization(JqColorization::Custom("0;34:0;34:0;34:0;34:0;34:0;34:0;34:0;34")), + JqOptions::default().with_colorization(JqColorization::Custom( + "0;34:0;34:0;34:0;34:0;34:0;34:0;34:0;34", + )), ); let expected = "\u{1b}[0;34m{\u{1b}[0m\u{1b}[0;34m\"name\"\u{1b}[0m\u{1b}[0;34m:\u{1b}[0m\u{1b}[0;34m\"john\"\u{1b}[0m\u{1b}[0;34m}\u{1b}[0m\n"; assert_eq!(res.unwrap(), expected); @@ -453,7 +457,20 @@ mod test { r#"{"c":"fourth","b":{"c":"third","a": "second"},"a":"first"}"#, JqOptions::default().with_sort_keys(true), ); - let expected = "{\"a\":\"first\",\"b\":{\"a\":\"second\",\"c\":\"third\"},\"c\":\"fourth\"}\n"; + let expected = + "{\"a\":\"first\",\"b\":{\"a\":\"second\",\"c\":\"third\"},\"c\":\"fourth\"}\n"; + assert_eq!(res.unwrap(), expected); + } + + #[test] + fn raw_input() { + let res = run_advanced( + r#"."#, + r#"{"name":"john"}"#, + JqOptions::default().with_raw_input(true), + ); + let expected = r#""{\"name\":\"john\"}" +"#; assert_eq!(res.unwrap(), expected); } diff --git a/src/options.rs b/src/options.rs index c9b5f34..9a457d4 100644 --- a/src/options.rs +++ b/src/options.rs @@ -15,7 +15,7 @@ pub struct JqOptions<'a> { impl<'a> Default for JqOptions<'a> { fn default() -> Self { JqOptions { - raw_input: false, // TODO + raw_input: false, raw_output: false, sort_keys: false, indentation: JqIndentation::Compact, From 18222c8e2c97b26fbd0067f34a030ca4ce2d289d Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sun, 28 Apr 2024 17:00:07 +0200 Subject: [PATCH 6/6] Documentation --- README.md | 88 +++++++++++++++++++++++++++++++++++++++--- examples/simple-cli.rs | 8 +++- src/lib.rs | 2 +- 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 74e5e8b..b0c6478 100644 --- a/README.md +++ b/README.md @@ -116,12 +116,90 @@ let parsed: Vec = serde_json::from_str(&output).unwrap(); assert_eq!(vec![2009, 2012, 2014, 2016, 2019], parsed); ``` +## Options -Barely any of the options or flags available from the [jq] cli are exposed -currently. -Literally all that is provided is the ability to execute a _jq program_ on a blob -of json. -Please pardon my dust as I sort out the details. +Jq has flags to alter the way in which data is input or output, some of these flags are supported. +The supported flags are available throught the _advanced varients of the run functions. + +```rust +use jq_rs; +use serde_json::{self, json}; + +let data = json!({ "title": "Coraline", "year": 2009 }); +let query = ".title"; + +// program output as a raw string, without quotes +let output = jq_rs::run_advanced(query, &data.to_string(), jq_rs::JqOptions::default().with_raw_output(true)); + +let output_raw = jq_rs::run_advanced(query, &data.to_string()); + +assert_eq!("\"Coraline\"", output); +``` + +### Raw input and raw output + +jq-rs supports the `-R, --raw-input` and `-r, --raw-output` flags through the following options: + +```rust +use jq_rs; +let options = jq_rs::JqOptions::default() + .with_raw_output(true) + .with_raw_input(true); +``` + +These are disabled by default. + +### Compact output + +jq-rs supports the `-c, --compact-output`, `--tabs` and `--indent n` flags through the following options: + +```rust +use jq_rs; +let compact = jq_rs::JqOptions::default() + .with_indentation(jq_rs::JqIndentation::Compact); + +let tabs = jq_rs::JqOptions::default() + .with_indentation(jq_rs::JqIndentation::Tabs); + +let spaces_2 = jq_rs::JqOptions::default() + .with_indentation(jq_rs::JqIndentation::Spaces(2)); +``` + +Compact is the default for this option. + +### Sorting + +jq-rs supports the `-S, --sort-keys` flag using the following option: + +```rust +use jq_rs; +let sorted = jq_rs::JqOptions::default() + .with_sort_keys(true); +``` + +Sorting is disabled by default. + +### Colorization + +jq-rs supports the `-C, --color-output` and `-M, --monochrome-output` flags. +jq-rs also supports custom colors, which are normally available through the `JQ_COLORS` environment variable. + +```rust +use jq_rs; + +let monochrome = jq_rs::JqOptions::default() + .with_colorization(jq_rs::JqColorization::Monochrome); + +let colorize = jq_rs::JqOptions::default() + .with_colorization(jq_rs::JqColorization::Colorize), + +let all_blue = jq_rs::JqOptions::default() + .with_colorization(jq_rs::JqColorization::Custom( + "0;34:0;34:0;34:0;34:0;34:0;34:0;34:0;34", + )); +``` + +The default option is monochrome, refer to the [jq documentation](https://jqlang.github.io/jq/manual/#colors) for using custom colors. ## Linking to libjq diff --git a/examples/simple-cli.rs b/examples/simple-cli.rs index a1717e4..6341abd 100644 --- a/examples/simple-cli.rs +++ b/examples/simple-cli.rs @@ -6,7 +6,13 @@ fn main() { let program = args.next().expect("jq program"); let input = args.next().expect("data input"); - match jq_rs::run(&program, &input) { + match jq_rs::run_advanced( + &program, + &input, + jq_rs::JqOptions::default() + .with_colorization(jq_rs::JqColorization::Colorize) + .with_indentation(jq_rs::JqIndentation::Spaces(2)), + ) { Ok(s) => print!("{}", s), // The output will include a trailing newline Err(e) => eprintln!("{}", e), } diff --git a/src/lib.rs b/src/lib.rs index 4a98c6c..45ff333 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -155,7 +155,7 @@ mod options; use std::ffi::CString; pub use errors::{Error, Result}; -pub use options::JqOptions; +pub use options::{JqOptions, JqColorization, JqIndentation}; /// Run a jq program on a blob of json data. ///