diff --git a/Cargo.lock b/Cargo.lock index 78c1948243274..3e7ab27c37bfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3354,6 +3354,7 @@ dependencies = [ "strum_macros 0.24.3", "unindent", "url", + "zmij", ] [[package]] @@ -3706,6 +3707,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "zmij", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index aa711f587cab0..67558735a753e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -461,6 +461,7 @@ wkt = "0.11.1" xorfilter-rs = { version = "0.5", features = ["cbordata"] } zerocopy = { version = "0.8.26", features = ["derive"] } zip = "3.0.0" +zmij = "1.0" zstd = "0.12.3" # AST needed diff --git a/src/query/ast/Cargo.toml b/src/query/ast/Cargo.toml index 2fbab0490bed9..2dacda6489be3 100644 --- a/src/query/ast/Cargo.toml +++ b/src/query/ast/Cargo.toml @@ -33,6 +33,7 @@ strum = { workspace = true } strum_macros = { workspace = true } unindent = { workspace = true } url = { workspace = true } +zmij = { workspace = true } [dev-dependencies] divan = { workspace = true } diff --git a/src/query/ast/src/ast/expr.rs b/src/query/ast/src/ast/expr.rs index fb5f907f523bb..160773ffc58c3 100644 --- a/src/query/ast/src/ast/expr.rs +++ b/src/query/ast/src/ast/expr.rs @@ -1069,7 +1069,9 @@ impl Display for Literal { } else if val.is_nan() { write!(f, "'NaN'::FLOAT64") } else { - write!(f, "{val}") + let mut buffer = zmij::Buffer::new(); + let s = buffer.format_finite(*val); + write!(f, "{s}") } } Literal::String(val) => { diff --git a/src/query/ast/src/parser/expr.rs b/src/query/ast/src/parser/expr.rs index 8fdf2a1718d0d..99089dcdd3081 100644 --- a/src/query/ast/src/parser/expr.rs +++ b/src/query/ast/src/parser/expr.rs @@ -2782,17 +2782,25 @@ pub fn parse_float(text: &str) -> Result { None => 0, }; - let p = i_part.len() as i32 + exp - f_part.len() as i32; - if !(-76..=76).contains(&p) { + let i_part_len = i_part.len() as i32; + let f_part_len = f_part.len() as i32; + let mut precision = i_part_len + f_part_len; + if exp > f_part_len { + precision += exp - f_part_len; + } else if i_part_len + exp < 0 { + precision -= i_part_len + exp; + } + + if precision > 76 { Ok(Literal::Float64(fast_float2::parse(text)?)) } else { - let mut digits = String::with_capacity(76); + let mut digits = String::with_capacity(precision as usize); digits.push_str(i_part); digits.push_str(f_part); if digits.is_empty() { digits.push('0') } - let mut scale = f_part.len() as i32 - exp; + let mut scale = f_part_len - exp; if scale < 0 { // e.g 123.1e3 for _ in 0..(-scale) { @@ -2800,13 +2808,6 @@ pub fn parse_float(text: &str) -> Result { } scale = 0; }; - - // truncate - if digits.len() > 76 { - scale -= digits.len() as i32 - 76; - digits.truncate(76); - } - Ok(Literal::Decimal256 { value: i256::from_str_radix(&digits, 10)?, precision: 76, diff --git a/src/query/formats/Cargo.toml b/src/query/formats/Cargo.toml index 6e341887d6b8d..01d42ed8e3f1f 100644 --- a/src/query/formats/Cargo.toml +++ b/src/query/formats/Cargo.toml @@ -36,6 +36,7 @@ num = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +zmij = { workspace = true } [dev-dependencies] diff --git a/src/query/formats/src/field_encoder/helpers/number_helpers.rs b/src/query/formats/src/field_encoder/helpers/number_helpers.rs index 4fc4a6b44c310..4263b93c5b3b9 100644 --- a/src/query/formats/src/field_encoder/helpers/number_helpers.rs +++ b/src/query/formats/src/field_encoder/helpers/number_helpers.rs @@ -47,7 +47,9 @@ macro_rules! impl_float { if self.is_nan() { buf.extend_from_slice(&settings.nan_bytes); } else { - extend_lexical(self, buf); + let mut buffer = zmij::Buffer::new(); + let s = buffer.format_finite(self); + buf.extend_from_slice(s.as_bytes()); } } } diff --git a/tests/sqllogictests/suites/base/11_data_type/11_0000_data_type_numeric.test b/tests/sqllogictests/suites/base/11_data_type/11_0000_data_type_numeric.test index 18d3ba498214a..2de7e78c987e1 100644 --- a/tests/sqllogictests/suites/base/11_data_type/11_0000_data_type_numeric.test +++ b/tests/sqllogictests/suites/base/11_data_type/11_0000_data_type_numeric.test @@ -45,6 +45,12 @@ SELECT * FROM t ---- -128 0 +onlyif mysql +query FF +select 10.234567899999999e80, 1234567890123456789012345678901234567890123456789012345678901234567890.1e-80; +---- +1.0234567899999998e+81 1.2345678901234568e-11 + statement ok DROP DATABASE data_type diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test index 5116724d2702b..4f4b9fee0d059 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test @@ -1243,7 +1243,7 @@ drop table if exists t_epoch; query T select EXTRACT(EPOCH FROM to_datetime('1969-12-31 23:59:59.999999')) ---- --1.0e-6 +-1e-6 query B select EXTRACT(EPOCH FROM now()) = epoch(now()) diff --git a/tests/sqllogictests/suites/query/functions/02_0014_function_maths.test b/tests/sqllogictests/suites/query/functions/02_0014_function_maths.test index 3907a7c6a6ff3..b84ed6ff0c4ff 100644 --- a/tests/sqllogictests/suites/query/functions/02_0014_function_maths.test +++ b/tests/sqllogictests/suites/query/functions/02_0014_function_maths.test @@ -387,8 +387,7 @@ SELECT (FLOOR(EXP(SQRT(1234.56789)) * 10000) % 18446744073709551615) b, (FLOOR(PI() * 10000) % 18446744073709551615) c, a + b - c ---- -9.754610558146624e15 1.817827108747062e19 31415.0 1.8188025698028737e19 - +9754610558146624.0 1.817827108747062e+19 31415.0 1.8188025698028737e+19 query F SELECT power(-2, 2) @@ -434,7 +433,7 @@ SELECT (FLOOR(EXP(SQRT(1234.56789)) * 10000) % 18446744073709551615) b, (FLOOR(PI() * 10000) % 18446744073709551615) c, a + b - c ---- -9.754610558146624e15 1.817827108747062e19 31415.0 1.8188025698028737e19 +9754610558146624.0 1.817827108747062e+19 31415.0 1.8188025698028737e+19 query T SELECT TRUNC(10.6); diff --git a/tests/sqllogictests/suites/query/functions/02_0060_function_geo.test b/tests/sqllogictests/suites/query/functions/02_0060_function_geo.test index c9970bf5db3de..d7a1d7f6dd6ed 100644 --- a/tests/sqllogictests/suites/query/functions/02_0060_function_geo.test +++ b/tests/sqllogictests/suites/query/functions/02_0060_function_geo.test @@ -76,4 +76,4 @@ select point_in_polygon((2.5,), [(4., 0.), (8., 4.), (4., 8.), (0., 4.)], [(3., query T select great_circle_angle(-2181569507.9714413, 15253014773.129665, 0.5823419941455749, 0.5823419941455749) ---- -1.3941863e12 \ No newline at end of file +1394186300000.0 diff --git a/tests/sqllogictests/suites/query/functions/02_0060_function_geo_h3.test b/tests/sqllogictests/suites/query/functions/02_0060_function_geo_h3.test index 4ff4ee573c6d2..7faad8590d0ec 100644 --- a/tests/sqllogictests/suites/query/functions/02_0060_function_geo_h3.test +++ b/tests/sqllogictests/suites/query/functions/02_0060_function_geo_h3.test @@ -120,7 +120,7 @@ select h3_get_base_cell(644325524701193974) query F select h3_hex_area_m2(1) ---- -6.097884417941339e11 +609788441794.1339 query F select h3_hex_area_km2(1) @@ -154,9 +154,9 @@ query F select h3_hex_area_m2(res) as m2 from t2 order by m2 ---- 1770347654.491309 -1.239343465508818e10 -8.680178039899731e10 -6.097884417941339e11 +12393434655.08818 +86801780398.99731 +609788441794.1339 query F select h3_hex_area_km2(res) as km2 from t2 order by km2 diff --git a/tests/sqllogictests/suites/stage/formats/avro/avro_copy.test b/tests/sqllogictests/suites/stage/formats/avro/avro_copy.test index b222022649c45..0784af1dabbf1 100644 --- a/tests/sqllogictests/suites/stage/formats/avro/avro_copy.test +++ b/tests/sqllogictests/suites/stage/formats/avro/avro_copy.test @@ -86,5 +86,5 @@ avro/number.avro 2 0 NULL NULL query select * from t ---- -2147483647 9223372036854775807 3.4028235e38 1.7976931348623157e308 --2147483648 -9223372036854775808 -3.4028235e38 -1.7976931348623157e308 +2147483647 9223372036854775807 3.4028235e+38 1.7976931348623157e+308 +-2147483648 -9223372036854775808 -3.4028235e+38 -1.7976931348623157e+308 diff --git a/tests/sqllogictests/suites/stage/formats/csv/csv_copy.test b/tests/sqllogictests/suites/stage/formats/csv/csv_copy.test index 9f29b1ea2cb22..115ee35a0161f 100644 --- a/tests/sqllogictests/suites/stage/formats/csv/csv_copy.test +++ b/tests/sqllogictests/suites/stage/formats/csv/csv_copy.test @@ -70,4 +70,4 @@ select * from t2 100.2345 200.789 300.0 -3.698114021686886e16 +3.698114021686886e+16