From a240742baef5497e1db4e32237c0479041fec297 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 15:11:47 +0000 Subject: [PATCH 01/11] repr: check the kind tag when decoding fixed-size column stats The Time, Timestamp, TimestampTz, Interval, and Uuid arms of col_values matched any FixedSize stats. from_bytes validates length only, and PackedNaiveDateTime, PackedInterval, and Uuid are all 16 bytes, so wrong-kind bytes decoded silently into garbage bounds. Not reachable today because a column's type cannot change under a reused name, but one schema evolution feature away from wrong bounds on old parts. Mismatched kinds now fall through to the catch-all arm and degrade to no stats. Also soften the Timestamp arm's two hard expects on the roundtrip through CheckedTimestamp to match the TimestampTz arm: malformed bytes should degrade, not panic the replica. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/repr/src/stats.rs | 58 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/src/repr/src/stats.rs b/src/repr/src/stats.rs index 21f39527c137c..b85f60a7be972 100644 --- a/src/repr/src/stats.rs +++ b/src/repr/src/stats.rs @@ -210,26 +210,50 @@ pub fn col_values<'a>( let upper = soft_expect_or_log(Date::from_pg_epoch(stats.upper))?; Some((Datum::Date(lower), Datum::Date(upper))) } - (SqlScalarType::Time, ColumnStatKinds::Bytes(BytesStats::FixedSize(stats))) => { + // NOTE: the `kind` field is checked in each fixed-size arm below + // because `from_bytes` validates length only, and PackedNaiveDateTime, + // PackedInterval, and Uuid are all 16 bytes: wrong-kind bytes would + // otherwise silently decode into garbage bounds. A mismatched kind + // falls through to the catch-all arm, which degrades to "no stats". + ( + SqlScalarType::Time, + ColumnStatKinds::Bytes(BytesStats::FixedSize( + stats @ FixedSizeBytesStats { + kind: FixedSizeBytesStatsKind::PackedTime, + .. + }, + )), + ) => { let lower = soft_expect_or_log(PackedNaiveTime::from_bytes(&stats.lower))?.into_value(); let upper = soft_expect_or_log(PackedNaiveTime::from_bytes(&stats.upper))?.into_value(); Some((Datum::Time(lower), Datum::Time(upper))) } - (SqlScalarType::Timestamp { .. }, ColumnStatKinds::Bytes(BytesStats::FixedSize(stats))) => { + ( + SqlScalarType::Timestamp { .. }, + ColumnStatKinds::Bytes(BytesStats::FixedSize( + stats @ FixedSizeBytesStats { + kind: FixedSizeBytesStatsKind::PackedDateTime, + .. + }, + )), + ) => { let lower = soft_expect_or_log(PackedNaiveDateTime::from_bytes(&stats.lower))?.into_value(); - let lower = - CheckedTimestamp::from_timestamplike(lower).expect("failed to roundtrip timestamp"); + let lower = soft_expect_or_log(CheckedTimestamp::from_timestamplike(lower))?; let upper = soft_expect_or_log(PackedNaiveDateTime::from_bytes(&stats.upper))?.into_value(); - let upper = - CheckedTimestamp::from_timestamplike(upper).expect("failed to roundtrip timestamp"); + let upper = soft_expect_or_log(CheckedTimestamp::from_timestamplike(upper))?; Some((Datum::Timestamp(lower), Datum::Timestamp(upper))) } ( SqlScalarType::TimestampTz { .. }, - ColumnStatKinds::Bytes(BytesStats::FixedSize(stats)), + ColumnStatKinds::Bytes(BytesStats::FixedSize( + stats @ FixedSizeBytesStats { + kind: FixedSizeBytesStatsKind::PackedDateTime, + .. + }, + )), ) => { let lower = soft_expect_or_log(PackedNaiveDateTime::from_bytes(&stats.lower))? .into_value() @@ -245,12 +269,28 @@ pub fn col_values<'a>( (SqlScalarType::MzTimestamp, ColumnStatKinds::Primitive(U64(stats))) => { map_stats(stats, |x| Datum::MzTimestamp(crate::Timestamp::from(x))) } - (SqlScalarType::Interval, ColumnStatKinds::Bytes(BytesStats::FixedSize(stats))) => { + ( + SqlScalarType::Interval, + ColumnStatKinds::Bytes(BytesStats::FixedSize( + stats @ FixedSizeBytesStats { + kind: FixedSizeBytesStatsKind::PackedInterval, + .. + }, + )), + ) => { let lower = soft_expect_or_log(PackedInterval::from_bytes(&stats.lower))?.into_value(); let upper = soft_expect_or_log(PackedInterval::from_bytes(&stats.upper))?.into_value(); Some((Datum::Interval(lower), Datum::Interval(upper))) } - (SqlScalarType::Uuid, ColumnStatKinds::Bytes(BytesStats::FixedSize(stats))) => { + ( + SqlScalarType::Uuid, + ColumnStatKinds::Bytes(BytesStats::FixedSize( + stats @ FixedSizeBytesStats { + kind: FixedSizeBytesStatsKind::Uuid, + .. + }, + )), + ) => { let lower = soft_expect_or_log(Uuid::from_slice(&stats.lower))?; let upper = soft_expect_or_log(Uuid::from_slice(&stats.upper))?; Some((Datum::Uuid(lower), Datum::Uuid(upper))) From 84034f570e93f1c66d00965325a3943aff06b54b Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 15:12:07 +0000 Subject: [PATCH 02/11] repr: validate V0 column stats instead of panicking on them The legacy Atomic arm of col_values used four hard expects on ProtoDatum decode: malformed V0 stats bytes panicked the replica at stats read time. It also never validated the decoded datum against the column type, and the V0 encoding carries no type tag, so a wrong-typed bound produced a range that excludes every value of the column's actual type, i.e. wrong results from corrupt legacy stats. Decode failures and type mismatches now degrade to no stats via soft errors. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/repr/src/stats.rs | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/src/repr/src/stats.rs b/src/repr/src/stats.rs index b85f60a7be972..667595babb4f8 100644 --- a/src/repr/src/stats.rs +++ b/src/repr/src/stats.rs @@ -41,7 +41,7 @@ use crate::adt::jsonb::{KeyClass, KeyClassifier, NumberParser}; use crate::adt::numeric::{Numeric, PackedNumeric}; use crate::adt::timestamp::{CheckedTimestamp, PackedNaiveDateTime}; use crate::row::ProtoDatum; -use crate::{Datum, RowArena, SqlScalarType}; +use crate::{Datum, Row, RowArena, SqlScalarType}; fn soft_expect_or_log(result: Result) -> Option { match result { @@ -319,16 +319,37 @@ pub fn col_values<'a>( | SqlScalarType::Uuid, ColumnStatKinds::Bytes(BytesStats::Atomic(AtomicBytesStats { lower, upper })), ) => { - let lower = ProtoDatum::decode(lower.as_slice()).expect("should be a valid ProtoDatum"); - let lower = arena.make_datum(|p| { - p.try_push_proto(&lower) - .expect("ProtoDatum should be valid Datum") - }); - let upper = ProtoDatum::decode(upper.as_slice()).expect("should be a valid ProtoDatum"); - let upper = arena.make_datum(|p| { - p.try_push_proto(&upper) - .expect("ProtoDatum should be valid Datum") - }); + // The V0 encoding carries no type tag, so a decoded bound has to + // be validated against the column type before it is used: a + // wrong-typed bound would produce a range that excludes every + // value of the column's actual type. Malformed or mismatched + // legacy bytes degrade to "no stats" instead of panicking. + fn decode_v0<'a>( + bytes: &[u8], + typ: &SqlScalarType, + arena: &'a RowArena, + ) -> Option> { + let proto = soft_expect_or_log(ProtoDatum::decode(bytes))?; + let mut row = Row::default(); + soft_expect_or_log(row.packer().try_push_proto(&proto))?; + let datum = arena.push_unary_row(row); + let type_matches = matches!( + (typ, datum), + (SqlScalarType::Numeric { .. }, Datum::Numeric(_)) + | (SqlScalarType::Time, Datum::Time(_)) + | (SqlScalarType::Timestamp { .. }, Datum::Timestamp(_)) + | (SqlScalarType::TimestampTz { .. }, Datum::TimestampTz(_)) + | (SqlScalarType::Interval, Datum::Interval(_)) + | (SqlScalarType::Uuid, Datum::Uuid(_)) + ); + if !type_matches { + soft_panic_or_log!("V0 stats bound {datum:?} does not match column {typ:?}"); + return None; + } + Some(datum) + } + let lower = decode_v0(lower.as_slice(), typ, arena)?; + let upper = decode_v0(upper.as_slice(), typ, arena)?; Some((lower, upper)) } From f8bc948046e1a4e2e9786280e356130b4d8ad8e1 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 15:12:32 +0000 Subject: [PATCH 03/11] storage-types: treat missing err stats as may-error in may_match_mfp may_match_mfp treated an absent err column in the part stats as "no errors", while the storage read path's filter_result treats it as "may error". The err column is on the default force-keep list for stats trimming, but that list is configurable, so a trimmed part would have let the peek path (StatsCursor uses may_match_mfp) skip a part whose error rows must surface regardless of any filter. Align with filter_result and keep the part. The regression test builds a part with an error row, strips the err column's stats, and applies a filter no Ok row matches: the old code skipped the part. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/storage-types/src/stats.rs | 63 ++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/storage-types/src/stats.rs b/src/storage-types/src/stats.rs index 39b8d8a7b5560..37155c1eb65c2 100644 --- a/src/storage-types/src/stats.rs +++ b/src/storage-types/src/stats.rs @@ -50,8 +50,10 @@ impl RelationPartStats<'_> { let mut ranges = ColumnSpecs::new(&relation, &arena); ranges.push_unmaterializable(UnmaterializableFunc::MzNow, time_range); - if self.err_count().into_iter().any(|count| count > 0) { - // If the error collection is nonempty, we always keep the part. + // If the error collection is nonempty, we always keep the part. + // Missing err stats mean errors cannot be ruled out, so they count as + // "may error" too, matching the storage read path's `filter_result`. + if self.err_count().is_none_or(|count| count > 0) { return true; } @@ -317,6 +319,63 @@ mod tests { ) } + /// The err column's stats are force-kept from trimming by default, but + /// that list is configurable, so they can be absent. When they are, + /// `may_match_mfp` must treat the part as possibly containing errors, + /// exactly like `filter_result` does: error rows must surface regardless + /// of any filter, so a part that may hold one can never be skipped. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // too slow + fn may_match_mfp_missing_err_stats_keeps_part() { + use mz_expr::{BinaryFunc, EvalError, MirScalarExpr, func}; + use mz_repr::ReprScalarType; + + use crate::errors::DataflowError; + + let schema = RelationDesc::builder() + .with_column("col", SqlScalarType::Int32.nullable(false)) + .finish(); + let mut builder = PartBuilder::new(&schema, &UnitSchema); + builder.push( + &SourceData(Ok(Row::pack_slice(&[Datum::Int32(1)]))), + &(), + 1u64, + 1i64, + ); + builder.push( + &SourceData(Err(DataflowError::from(EvalError::DivisionByZero))), + &(), + 1u64, + 1i64, + ); + let part = builder.finish(); + let key_col = part.key.as_struct(); + let decoder = >::decoder(&schema, key_col.clone()) + .expect("success"); + let mut key_stats = decoder.stats(); + // Simulate the err column's stats having been trimmed away. + key_stats.cols.remove("err").expect("err stats present"); + + let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); + let stats = RelationPartStats { + name: "test", + metrics: &metrics, + stats: &PartStats { key: key_stats }, + desc: &schema, + }; + // No Ok row matches this filter: only the error row makes the part + // relevant, and with the err stats missing it cannot be ruled out. + let mfp = MapFilterProject::new(1).filter(std::iter::once(MirScalarExpr::CallBinary { + func: BinaryFunc::Eq(func::Eq), + expr1: Box::new(MirScalarExpr::column(0)), + expr2: Box::new(MirScalarExpr::literal_ok( + Datum::Int32(999), + ReprScalarType::Int32, + )), + })); + assert!(stats.may_match_mfp(ResultSpec::anything(), &mfp)); + } + #[mz_ore::test] #[ignore] // TODO(parkmycar): Re-enable this test with a smaller sample size. fn statistics_stability() { From f1ec4400b64650bd6b73d966fa48249569e2e184 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 20:33:25 +0000 Subject: [PATCH 04/11] persist: fail open when part stats do not decode The shard_source filter and the stats accessors in fetch called LazyPartStats::decode, which panics on undecodable bytes, on stats that other processes wrote. Stats from a newer version can use a proto variant an older reader does not know, so a reader inside the upgrade window panicked in the pushdown filter instead of falling back to fetching the part, the fail-open behavior every other missing-stats case on this path already has. Decode failures now degrade to "no stats". Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/persist-client/src/fetch.rs | 12 +++++++-- src/persist-client/src/internal/encoding.rs | 25 +++++++++++++++++++ .../src/operators/shard_source.rs | 14 ++++++++++- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/persist-client/src/fetch.rs b/src/persist-client/src/fetch.rs index ead65902a5d1f..4b95eb9c8344b 100644 --- a/src/persist-client/src/fetch.rs +++ b/src/persist-client/src/fetch.rs @@ -656,8 +656,11 @@ where } /// Returns the pushdown stats for this part. + /// + /// Stats written by a newer version may not decode; those return `None`, + /// the same as a part that carries no stats. pub fn stats(&self) -> Option { - self.part.stats().map(|x| x.decode()) + self.part.stats().and_then(|x| x.try_decode().ok()) } /// Apply any relevant projection pushdown optimizations, assuming that the data in the part @@ -870,9 +873,14 @@ impl FetchedBlob Option { match &self.buf { - FetchedBlobBuf::Hollow { part, .. } => part.stats.as_ref().map(|x| x.decode()), + FetchedBlobBuf::Hollow { part, .. } => { + part.stats.as_ref().and_then(|x| x.try_decode().ok()) + } FetchedBlobBuf::Inline { .. } => None, } } diff --git a/src/persist-client/src/internal/encoding.rs b/src/persist-client/src/internal/encoding.rs index 0005b7b642111..8819c0f2bec73 100644 --- a/src/persist-client/src/internal/encoding.rs +++ b/src/persist-client/src/internal/encoding.rs @@ -1977,6 +1977,7 @@ impl RustType for Antichain { #[cfg(test)] mod tests { use mz_ore::assert_none; + use mz_persist_types::stats::{ProtoDynStats, ProtoStructStats}; use bytes::Bytes; use mz_build_info::DUMMY_BUILD_INFO; @@ -2666,4 +2667,28 @@ mod tests { assert_err!(stats.try_decode()); assert!(format!("{stats:?}").contains("undecodable")); } + + /// The exact shape version skew produces: valid protobuf whose stats + /// oneof uses a variant this version does not know (a newer writer's new + /// stats kind reaching an older reader). + fn version_skewed_part_stats() -> LazyPartStats { + let mut proto = ProtoStructStats::default(); + proto.cols.insert("c".into(), ProtoDynStats::default()); + let bytes = prost::Message::encode_to_vec(&proto); + LazyPartStats::from_proto(Bytes::from(bytes)).expect("stats bytes are stored undecoded") + } + + /// `decode` panics on stats from a newer version, which is why the read + /// paths (the shard_source filter and the `stats()` accessors in fetch) + /// must use `try_decode` and fail open to fetching the part. + #[mz_ore::test] + #[should_panic(expected = "valid stats")] + fn part_stats_decode_panics_on_unknown_variant() { + let _ = version_skewed_part_stats().decode(); + } + + #[mz_ore::test] + fn part_stats_try_decode_fails_open_on_unknown_variant() { + assert_err!(version_skewed_part_stats().try_decode()); + } } diff --git a/src/persist-client/src/operators/shard_source.rs b/src/persist-client/src/operators/shard_source.rs index 2d100cd4b75ad..fa5efad69df75 100644 --- a/src/persist-client/src/operators/shard_source.rs +++ b/src/persist-client/src/operators/shard_source.rs @@ -534,7 +534,19 @@ where BatchPart::Hollow(x) => { let should_fetch = x.stats.as_ref().map_or(FilterResult::Keep, |stats| { - filter_fn(&stats.decode(), current_frontier.borrow()) + // Stats written by a newer version may + // not decode. The sound fallback is to + // fetch the part. + match stats.try_decode() { + Ok(stats) => filter_fn(&stats, current_frontier.borrow()), + Err(err) => { + tracing::warn!( + %err, + "could not decode part stats, fetching part" + ); + FilterResult::Keep + } + } }); should_fetch } From 2922cab4976f099ed17616ce7827a6bb324c5018 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 20:35:43 +0000 Subject: [PATCH 05/11] storage-types: treat malformed err stats as unknown err count ok_count had a hard expect on the err column's stats shape: corrupt or version-skewed durable stats panicked the replica at pushdown time. An unknown err count already fails open (callers keep the part), so degrade to that instead. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/storage-types/src/stats.rs | 57 ++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/src/storage-types/src/stats.rs b/src/storage-types/src/stats.rs index 37155c1eb65c2..8858b99ad46f9 100644 --- a/src/storage-types/src/stats.rs +++ b/src/storage-types/src/stats.rs @@ -176,12 +176,10 @@ impl RelationPartStats<'_> { pub fn ok_count(&self) -> Option { // The number of OKs is the number of rows whose error is None. - let stats = self - .stats - .key - .col("err")? - .try_as_optional_bytes() - .expect("err column should be a Option>"); + // Malformed or wrong-shaped err stats (corrupt or version-skewed + // durable state) count as unknown, which callers treat as + // "may contain errors", rather than panicking the replica. + let stats = self.stats.key.col("err")?.try_as_optional_bytes().ok()?; Some(stats.none) } @@ -376,6 +374,53 @@ mod tests { assert!(stats.may_match_mfp(ResultSpec::anything(), &mfp)); } + /// Wrong-shaped err-column stats (corrupt or version-skewed durable + /// state) must read as "err count unknown", which fails open to keeping + /// the part, not panic the replica. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // too slow + fn malformed_err_stats_fail_open() { + use mz_persist_types::stats::{ColumnNullStats, ColumnarStats, PrimitiveStats}; + + let schema = RelationDesc::builder() + .with_column("col", SqlScalarType::Int32.nullable(false)) + .finish(); + let mut builder = PartBuilder::new(&schema, &UnitSchema); + builder.push( + &SourceData(Ok(Row::pack_slice(&[Datum::Int32(1)]))), + &(), + 1u64, + 1i64, + ); + let part = builder.finish(); + let key_col = part.key.as_struct(); + let decoder = >::decoder(&schema, key_col.clone()) + .expect("success"); + let mut key_stats = decoder.stats(); + // Overwrite the err column's stats with a wrong-shaped entry. + key_stats.cols.insert( + "err".to_string(), + ColumnarStats { + nulls: Some(ColumnNullStats { count: 0 }), + values: PrimitiveStats { + lower: 0i32, + upper: 0i32, + } + .into(), + }, + ); + + let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); + let stats = RelationPartStats { + name: "test", + metrics: &metrics, + stats: &PartStats { key: key_stats }, + desc: &schema, + }; + assert_eq!(stats.ok_count(), None); + assert_eq!(stats.err_count(), None); + } + #[mz_ore::test] #[ignore] // TODO(parkmycar): Re-enable this test with a smaller sample size. fn statistics_stability() { From a746e2649f74b5a8c22e8e525468f4e646639735 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 20:40:16 +0000 Subject: [PATCH 06/11] repr, persist: guard the latent filter pushdown invariants apply_demand filters metadata by raw ColumnIndex but types by position, which only agree on dense descs. Assert that, so a future dropped-column desc fails loudly instead of attaching statistics and filter specs to the wrong columns. Document the two other invariants the residual-risk review found load-bearing but unwritten: name-keyed part stats, and diffs_sum substitution requiring registered batch bounds to match the blob. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/persist-client/src/fetch.rs | 5 +++++ src/repr/src/relation.rs | 30 ++++++++++++++++++++++++++++++ src/repr/src/row/encode.rs | 8 ++++++++ 3 files changed, 43 insertions(+) diff --git a/src/persist-client/src/fetch.rs b/src/persist-client/src/fetch.rs index 4b95eb9c8344b..1d5c956162654 100644 --- a/src/persist-client/src/fetch.rs +++ b/src/persist-client/src/fetch.rs @@ -691,6 +691,11 @@ where &[as_of] => as_of, _ => return, }; + // NOTE: `diffs_sum` sums every row physically in the blob, while + // reads truncate rows outside the registered desc. Substituting it is + // sound only while no writer registers a batch with tighter bounds + // than the blob holds (none does today, and rewritten batches prove + // it), which nothing here can re-check without fetching the blob. let eligible = self.desc.upper().less_equal(as_of) && self.desc.since().less_equal(as_of); if !eligible { return; diff --git a/src/repr/src/relation.rs b/src/repr/src/relation.rs index 6ca97bf9acb2f..51679b6b2aaa5 100644 --- a/src/repr/src/relation.rs +++ b/src/repr/src/relation.rs @@ -1399,6 +1399,19 @@ impl RelationDesc { /// Creates a new [`RelationDesc`] retaining only the columns specified in `demands`. pub fn apply_demand(&self, demands: &BTreeSet) -> RelationDesc { + // This filters `metadata` by raw ColumnIndex but `typ` by position, + // which only agree when the desc is dense. Every desc constructible + // today is (schema history is add-only), but a dropped column would + // desync the two and silently attach types, statistics, and filter + // specs to the wrong columns downstream. + debug_assert!( + self.metadata + .iter() + .enumerate() + .all(|(pos, (idx, meta))| idx.0 == pos && meta.typ_idx == pos), + "apply_demand requires a dense RelationDesc (ColumnIndex == typ_idx): {:?}", + self.metadata, + ); let mut new_desc = self.clone(); // Update ColumnMetadata. @@ -2066,6 +2079,23 @@ mod tests { use super::*; use prost::Message; + /// `apply_demand`, and the stats and filter-spec plumbing downstream of + /// it, require dense descs. A desc with a dropped column must trip the + /// assertion rather than silently misattach columns. + #[mz_ore::test] + #[should_panic(expected = "dense RelationDesc")] + fn apply_demand_rejects_non_dense_desc() { + let desc = RelationDesc::builder() + .with_column("a", SqlScalarType::Int32.nullable(false)) + .with_column("b", SqlScalarType::Int32.nullable(false)) + .with_column("c", SqlScalarType::Int32.nullable(false)) + .finish(); + let mut versioned = VersionedRelationDesc::new(desc); + let version = versioned.drop_column("b"); + let desc = versioned.at_version(RelationVersionSelector::Specific(version)); + let _ = desc.apply_demand(&BTreeSet::from([0])); + } + #[mz_ore::test] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `pipe2` on OS `linux` fn smoktest_at_version() { diff --git a/src/repr/src/row/encode.rs b/src/repr/src/row/encode.rs index 5a815f07ab37c..a0569c71c6321 100644 --- a/src/repr/src/row/encode.rs +++ b/src/repr/src/row/encode.rs @@ -1442,6 +1442,14 @@ impl RowColumnarEncoder { // We name the Fields in Parquet with the column index, but for // backwards compat use the column name for stats. + // + // NOTE: name-keyed stats are sound only while durable + // relations never carry duplicate column names (the planner + // enforces this) and a dropped column's name can never be + // reused by a later version (persist rejects schema + // migrations containing drops). Filter pushdown consults + // these stats by name; violating either invariant attaches + // one column's stats to another and yields wrong results. let name = (col_idx.to_raw(), col_name.as_str().into()); (name, encoder) From e0e4e348605bcb1e16be3bbc36c74e537efb489d Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 20:41:47 +0000 Subject: [PATCH 07/11] storage: audit error rows in wrongly-discarded parts The pushdown audit fired only when the MFP produced output on an Ok row. Error rows from an audited part were emitted with no check, so a part discarded because its err stats undercount, the one violation class the err-count guard exists for, passed the audit silently. Mirror the audit into the error arm. No deterministic test exists for this: the audit only fires on an actual violation, which requires a live stats bug to construct through the runtime operator. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/storage-operators/src/persist_source.rs | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/storage-operators/src/persist_source.rs b/src/storage-operators/src/persist_source.rs index 4d742a5897f76..700c088447da4 100644 --- a/src/storage-operators/src/persist_source.rs +++ b/src/storage-operators/src/persist_source.rs @@ -702,6 +702,30 @@ impl PendingWork { } } (SourceData(Err(err)), ()) => { + // A discarded part that turns out to hold an error row is + // as much a pushdown violation as one whose MFP yields + // output: errors must surface regardless of any filter. + // Without this arm the audit was blind to exactly the + // undercounted-err-stats violation class. + if let Some(stats) = &is_filter_pushdown_audit { + sentry::with_scope( + |scope| scope.set_tag("alert_id", "persist_pushdown_audit_violation"), + || { + error!( + ?stats, + name, + ?err, + "persist filter pushdown correctness violation!" + ); + if self.panic_on_audit_failure { + panic!( + "persist filter pushdown correctness violation! {}", + name + ); + } + }, + ); + } let mut emit_time = *self.capability.time(); emit_time.0 = time; session.give((Err(E::from(err)), emit_time, diff.into())); From d9f9f1c08bb0337aadffce9b83b9a7e1650f0dee Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 16:16:17 +0000 Subject: [PATCH 08/11] storage-types, expr, repr: systematic filter pushdown test coverage Corpus: add -NaN and -0.0 floats, jsonb maps and lists, truncation-edge strings, and a leap-second time to interesting_datums. Containment: check that stats-derived specs contain every datum for every scalar type, over datum pairs and full sets, after trim and trim_to_budget at every budget. End to end: wide-schema and multi-part audit proptests over real part stats with mz_now bounds and error rows, plus zero-column ReplaceWith and schema-drift cases, and regression tests for both PER-53 variants in pushdown.slt. Interpreter: grow the equivalence proptest vocabulary to 18 scalar types and ~150 declared-monotone functions, validating their monotonicity claims continuously. PROPTEST_CASES now overrides the built-in case counts for long runs, and a new coverage-guided pushdown_soundness cargo-fuzz target explores raw bit patterns beyond the corpus with the same soundness oracle. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/expr/src/interpret.rs | 396 ++++++++++- src/repr/src/scalar.rs | 46 +- .../proptest-regressions/persist_source.txt | 2 + src/storage-operators/src/persist_source.rs | 640 +++++++++++++++++- src/storage-types/fuzz/Cargo.toml | 11 + .../fuzz/fuzz_targets/pushdown_soundness.rs | 337 +++++++++ src/storage-types/src/stats.rs | 97 ++- test/cargo-fuzz/mzcompose.py | 39 +- test/pgtest-mz/datums.pt | 92 +-- 9 files changed, 1576 insertions(+), 84 deletions(-) create mode 100644 src/storage-types/fuzz/fuzz_targets/pushdown_soundness.rs diff --git a/src/expr/src/interpret.rs b/src/expr/src/interpret.rs index 27ea74ee1d1f4..eb14a0868062b 100644 --- a/src/expr/src/interpret.rs +++ b/src/expr/src/interpret.rs @@ -1393,11 +1393,18 @@ mod tests { ReprScalarType::Bool, ReprScalarType::Jsonb, NUM_TYPE, + ReprScalarType::Int16, ReprScalarType::Int32, + ReprScalarType::Int64, + ReprScalarType::UInt16, + ReprScalarType::UInt32, + ReprScalarType::UInt64, ReprScalarType::Float32, ReprScalarType::Float64, ReprScalarType::Date, + ReprScalarType::Time, ReprScalarType::Timestamp, + ReprScalarType::TimestampTz, ReprScalarType::MzTimestamp, ReprScalarType::Interval, ReprScalarType::String, @@ -1419,22 +1426,266 @@ mod tests { UnaryFunc::IsNull(IsNull), UnaryFunc::IsFalse(IsFalse), UnaryFunc::TryParseMonotonicIso8601Timestamp(TryParseMonotonicIso8601Timestamp), + // Declared-monotone functions whose claims are otherwise + // unvalidated, chosen for fallible or lossy interiors: the + // equivalence proptests catch a wrong claim as a spec that fails + // to contain the evaluated result. + UnaryFunc::NegInt32(NegInt32), + UnaryFunc::NegInt64(NegInt64), + UnaryFunc::CastInt32ToUint32(CastInt32ToUint32), + UnaryFunc::CastInt64ToInt32(CastInt64ToInt32), + UnaryFunc::CastInt64ToNumeric(CastInt64ToNumeric(None)), + UnaryFunc::CastFloat64ToInt64(CastFloat64ToInt64), + UnaryFunc::CastFloat64ToFloat32(CastFloat64ToFloat32), + UnaryFunc::CastFloat32ToFloat64(CastFloat32ToFloat64), + UnaryFunc::CastNumericToInt64(CastNumericToInt64), + UnaryFunc::CeilNumeric(CeilNumeric), + UnaryFunc::FloorNumeric(FloorNumeric), + UnaryFunc::CastDateToTimestamp(CastDateToTimestamp(None)), + UnaryFunc::CastTimestampToTimestampTz(CastTimestampToTimestampTz { + from: None, + to: None, + }), + UnaryFunc::CastTimestampTzToTimestamp(CastTimestampTzToTimestamp { + from: None, + to: None, + }), + // Conditionally monotone (most significant unit) and its + // non-monotone sibling. + UnaryFunc::ExtractTimestamp(ExtractTimestamp(DateTimeUnits::Year)), + UnaryFunc::ExtractTimestamp(ExtractTimestamp(DateTimeUnits::Month)), + UnaryFunc::ExtractTimestampTz(ExtractTimestampTz(DateTimeUnits::Epoch)), + UnaryFunc::ExtractTimestampTz(ExtractTimestampTz(DateTimeUnits::Year)), + // Batch 2 of the declared-monotone sweep: the remaining cast + // families, ordered-domain arithmetic helpers, and functions with + // partial domains (errors on part of the range). + UnaryFunc::CastBoolToInt32(CastBoolToInt32), + UnaryFunc::CastBoolToString(CastBoolToString), + UnaryFunc::NegInt16(NegInt16), + UnaryFunc::CastInt16ToInt32(CastInt16ToInt32), + UnaryFunc::CastInt16ToInt64(CastInt16ToInt64), + UnaryFunc::CastInt16ToFloat32(CastInt16ToFloat32), + UnaryFunc::CastInt16ToFloat64(CastInt16ToFloat64), + UnaryFunc::CastInt16ToUint16(CastInt16ToUint16), + UnaryFunc::CastInt16ToNumeric(CastInt16ToNumeric(None)), + UnaryFunc::CastInt32ToInt16(CastInt32ToInt16), + UnaryFunc::CastInt32ToInt64(CastInt32ToInt64), + UnaryFunc::CastInt32ToFloat32(CastInt32ToFloat32), + UnaryFunc::CastInt32ToFloat64(CastInt32ToFloat64), + UnaryFunc::CastInt32ToUint16(CastInt32ToUint16), + UnaryFunc::CastInt32ToNumeric(CastInt32ToNumeric(None)), + UnaryFunc::CastInt32ToMzTimestamp(CastInt32ToMzTimestamp), + UnaryFunc::CastInt64ToInt16(CastInt64ToInt16), + UnaryFunc::CastInt64ToFloat32(CastInt64ToFloat32), + UnaryFunc::CastInt64ToFloat64(CastInt64ToFloat64), + UnaryFunc::CastInt64ToUint64(CastInt64ToUint64), + UnaryFunc::CastInt64ToMzTimestamp(CastInt64ToMzTimestamp), + UnaryFunc::CastUint64ToUint32(CastUint64ToUint32), + UnaryFunc::CastUint64ToInt32(CastUint64ToInt32), + UnaryFunc::CastUint64ToNumeric(CastUint64ToNumeric(None)), + UnaryFunc::CastUint64ToMzTimestamp(CastUint64ToMzTimestamp), + UnaryFunc::NegFloat32(NegFloat32), + UnaryFunc::FloorFloat32(FloorFloat32), + UnaryFunc::CastFloat32ToInt32(CastFloat32ToInt32), + UnaryFunc::CastFloat32ToNumeric(CastFloat32ToNumeric(None)), + UnaryFunc::FloorFloat64(FloorFloat64), + UnaryFunc::CastFloat64ToInt32(CastFloat64ToInt32), + UnaryFunc::CastFloat64ToUint64(CastFloat64ToUint64), + UnaryFunc::CastFloat64ToNumeric(CastFloat64ToNumeric(None)), + UnaryFunc::RoundNumeric(RoundNumeric), + UnaryFunc::TruncNumeric(TruncNumeric), + UnaryFunc::Log10Numeric(Log10Numeric), + UnaryFunc::CastNumericToFloat64(CastNumericToFloat64), + UnaryFunc::CastNumericToInt32(CastNumericToInt32), + UnaryFunc::CastTimestampToDate(CastTimestampToDate), + UnaryFunc::CastDateToMzTimestamp(CastDateToMzTimestamp), + UnaryFunc::StepMzTimestamp(StepMzTimestamp), + // Batch 3: every remaining declared-monotone cast family, the + // anti-monotone bitwise complements, and the conditional + // most-significant-unit extracts for date and timestamptz. + UnaryFunc::CastBoolToStringNonstandard(CastBoolToStringNonstandard), + UnaryFunc::CastBoolToInt64(CastBoolToInt64), + UnaryFunc::CastInt16ToUint32(CastInt16ToUint32), + UnaryFunc::CastInt16ToUint64(CastInt16ToUint64), + UnaryFunc::CastInt32ToUint64(CastInt32ToUint64), + UnaryFunc::CastInt64ToUint16(CastInt64ToUint16), + UnaryFunc::CastInt64ToUint32(CastInt64ToUint32), + UnaryFunc::CastUint16ToUint32(CastUint16ToUint32), + UnaryFunc::CastUint16ToUint64(CastUint16ToUint64), + UnaryFunc::CastUint16ToInt16(CastUint16ToInt16), + UnaryFunc::CastUint16ToInt32(CastUint16ToInt32), + UnaryFunc::CastUint16ToFloat32(CastUint16ToFloat32), + UnaryFunc::CastUint16ToFloat64(CastUint16ToFloat64), + UnaryFunc::CastUint16ToNumeric(CastUint16ToNumeric(None)), + UnaryFunc::CastUint16ToInt64(CastUint16ToInt64), + UnaryFunc::BitNotUint16(BitNotUint16), + UnaryFunc::CastUint32ToUint16(CastUint32ToUint16), + UnaryFunc::CastUint32ToUint64(CastUint32ToUint64), + UnaryFunc::CastUint32ToInt32(CastUint32ToInt32), + UnaryFunc::CastUint32ToInt64(CastUint32ToInt64), + UnaryFunc::CastUint32ToFloat32(CastUint32ToFloat32), + UnaryFunc::CastUint32ToFloat64(CastUint32ToFloat64), + UnaryFunc::CastUint32ToNumeric(CastUint32ToNumeric(None)), + UnaryFunc::CastUint32ToInt16(CastUint32ToInt16), + UnaryFunc::CastUint32ToMzTimestamp(CastUint32ToMzTimestamp), + UnaryFunc::BitNotUint32(BitNotUint32), + UnaryFunc::CastUint64ToUint16(CastUint64ToUint16), + UnaryFunc::CastUint64ToInt16(CastUint64ToInt16), + UnaryFunc::CastUint64ToInt64(CastUint64ToInt64), + UnaryFunc::CastUint64ToFloat32(CastUint64ToFloat32), + UnaryFunc::CastUint64ToFloat64(CastUint64ToFloat64), + UnaryFunc::BitNotUint64(BitNotUint64), + UnaryFunc::CastFloat32ToInt16(CastFloat32ToInt16), + UnaryFunc::CastFloat32ToInt64(CastFloat32ToInt64), + UnaryFunc::CastFloat32ToUint16(CastFloat32ToUint16), + UnaryFunc::CastFloat32ToUint32(CastFloat32ToUint32), + UnaryFunc::CastFloat32ToUint64(CastFloat32ToUint64), + UnaryFunc::CastFloat64ToInt16(CastFloat64ToInt16), + UnaryFunc::CastFloat64ToUint16(CastFloat64ToUint16), + UnaryFunc::CastFloat64ToUint32(CastFloat64ToUint32), + UnaryFunc::CastJsonbToInt16(CastJsonbToInt16), + UnaryFunc::CastJsonbToInt32(CastJsonbToInt32), + UnaryFunc::CastJsonbToInt64(CastJsonbToInt64), + UnaryFunc::CastJsonbToFloat32(CastJsonbToFloat32), + UnaryFunc::CastJsonbToFloat64(CastJsonbToFloat64), + UnaryFunc::CastNumericToInt16(CastNumericToInt16), + UnaryFunc::CastNumericToFloat32(CastNumericToFloat32), + UnaryFunc::CastNumericToUint16(CastNumericToUint16), + UnaryFunc::CastNumericToUint32(CastNumericToUint32), + UnaryFunc::CastNumericToUint64(CastNumericToUint64), + UnaryFunc::CastTimestampTzToDate(CastTimestampTzToDate), + UnaryFunc::CastTimestampTzToMzTimestamp(CastTimestampTzToMzTimestamp), + UnaryFunc::DateTruncTimestampTz(DateTruncTimestampTz(DateTimeUnits::Epoch)), + UnaryFunc::CastDateToTimestampTz(CastDateToTimestampTz(None)), + UnaryFunc::ExtractDate(ExtractDate(DateTimeUnits::Year)), + UnaryFunc::ExtractDate(ExtractDate(DateTimeUnits::Day)), ] }; fn unary_typecheck(func: &UnaryFunc, arg: &ReprColumnType) -> bool { use UnaryFunc::*; match func { - CastNumericToMzTimestamp(_) | NegNumeric(_) => arg.scalar_type == NUM_TYPE, - NegFloat64(_) => arg.scalar_type == ReprScalarType::Float64, - CastTimestampToMzTimestamp(_) => arg.scalar_type == ReprScalarType::Timestamp, - CastJsonbToNumeric(_) | CastJsonbToBool(_) | CastJsonbToString(_) => { - arg.scalar_type == ReprScalarType::Jsonb - } + CastNumericToMzTimestamp(_) + | NegNumeric(_) + | CastNumericToInt64(_) + | CeilNumeric(_) + | FloorNumeric(_) + | RoundNumeric(_) + | TruncNumeric(_) + | Log10Numeric(_) + | CastNumericToFloat64(_) + | CastNumericToInt32(_) + | CastNumericToInt16(_) + | CastNumericToFloat32(_) + | CastNumericToUint16(_) + | CastNumericToUint32(_) + | CastNumericToUint64(_) => arg.scalar_type == NUM_TYPE, + NegFloat64(_) + | CastFloat64ToInt64(_) + | CastFloat64ToFloat32(_) + | FloorFloat64(_) + | CastFloat64ToInt32(_) + | CastFloat64ToUint64(_) + | CastFloat64ToNumeric(_) + | CastFloat64ToInt16(_) + | CastFloat64ToUint16(_) + | CastFloat64ToUint32(_) => arg.scalar_type == ReprScalarType::Float64, + CastFloat32ToFloat64(_) + | NegFloat32(_) + | FloorFloat32(_) + | CastFloat32ToInt32(_) + | CastFloat32ToNumeric(_) + | CastFloat32ToInt16(_) + | CastFloat32ToInt64(_) + | CastFloat32ToUint16(_) + | CastFloat32ToUint32(_) + | CastFloat32ToUint64(_) => arg.scalar_type == ReprScalarType::Float32, + NegInt16(_) + | CastInt16ToInt32(_) + | CastInt16ToInt64(_) + | CastInt16ToFloat32(_) + | CastInt16ToFloat64(_) + | CastInt16ToUint16(_) + | CastInt16ToNumeric(_) + | CastInt16ToUint32(_) + | CastInt16ToUint64(_) => arg.scalar_type == ReprScalarType::Int16, + NegInt32(_) + | CastInt32ToUint32(_) + | CastInt32ToInt16(_) + | CastInt32ToInt64(_) + | CastInt32ToFloat32(_) + | CastInt32ToFloat64(_) + | CastInt32ToUint16(_) + | CastInt32ToNumeric(_) + | CastInt32ToMzTimestamp(_) + | CastInt32ToUint64(_) => arg.scalar_type == ReprScalarType::Int32, + NegInt64(_) + | CastInt64ToInt32(_) + | CastInt64ToNumeric(_) + | CastInt64ToInt16(_) + | CastInt64ToFloat32(_) + | CastInt64ToFloat64(_) + | CastInt64ToUint64(_) + | CastInt64ToMzTimestamp(_) + | CastInt64ToUint16(_) + | CastInt64ToUint32(_) => arg.scalar_type == ReprScalarType::Int64, + CastUint16ToUint32(_) + | CastUint16ToUint64(_) + | CastUint16ToInt16(_) + | CastUint16ToInt32(_) + | CastUint16ToFloat32(_) + | CastUint16ToFloat64(_) + | CastUint16ToNumeric(_) + | CastUint16ToInt64(_) + | BitNotUint16(_) => arg.scalar_type == ReprScalarType::UInt16, + CastUint32ToUint16(_) + | CastUint32ToUint64(_) + | CastUint32ToInt32(_) + | CastUint32ToInt64(_) + | CastUint32ToFloat32(_) + | CastUint32ToFloat64(_) + | CastUint32ToNumeric(_) + | CastUint32ToInt16(_) + | CastUint32ToMzTimestamp(_) + | BitNotUint32(_) => arg.scalar_type == ReprScalarType::UInt32, + CastUint64ToUint32(_) + | CastUint64ToInt32(_) + | CastUint64ToNumeric(_) + | CastUint64ToMzTimestamp(_) + | CastUint64ToUint16(_) + | CastUint64ToInt16(_) + | CastUint64ToInt64(_) + | CastUint64ToFloat32(_) + | CastUint64ToFloat64(_) + | BitNotUint64(_) => arg.scalar_type == ReprScalarType::UInt64, + StepMzTimestamp(_) => arg.scalar_type == ReprScalarType::MzTimestamp, + CastBoolToInt32(_) + | CastBoolToString(_) + | CastBoolToStringNonstandard(_) + | CastBoolToInt64(_) => arg.scalar_type == ReprScalarType::Bool, + CastTimestampToMzTimestamp(_) + | CastTimestampToTimestampTz(_) + | CastTimestampToDate(_) => arg.scalar_type == ReprScalarType::Timestamp, + CastTimestampTzToTimestamp(_) + | ExtractTimestampTz(_) + | CastTimestampTzToDate(_) + | CastTimestampTzToMzTimestamp(_) + | DateTruncTimestampTz(_) => arg.scalar_type == ReprScalarType::TimestampTz, + CastJsonbToNumeric(_) + | CastJsonbToBool(_) + | CastJsonbToString(_) + | CastJsonbToInt16(_) + | CastJsonbToInt32(_) + | CastJsonbToInt64(_) + | CastJsonbToFloat32(_) + | CastJsonbToFloat64(_) => arg.scalar_type == ReprScalarType::Jsonb, ExtractTimestamp(_) | DateTruncTimestamp(_) => { arg.scalar_type == ReprScalarType::Timestamp } - ExtractDate(_) => arg.scalar_type == ReprScalarType::Date, + ExtractDate(_) + | CastDateToTimestamp(_) + | CastDateToMzTimestamp(_) + | CastDateToTimestampTz(_) => arg.scalar_type == ReprScalarType::Date, Not(_) => arg.scalar_type == ReprScalarType::Bool, IsNull(_) => true, TryParseMonotonicIso8601Timestamp(_) => arg.scalar_type == ReprScalarType::String, @@ -1464,6 +1715,52 @@ mod tests { DateTruncUnitsTimestamp.into(), JsonbGetString.into(), JsonbGetStringStringify.into(), + // Declared-monotone integer arithmetic: overflow and + // division-by-zero are interior error conditions the endpoints + // need not reveal. + AddInt32.into(), + SubInt32.into(), + MulInt32.into(), + DivInt32.into(), + AddInt64.into(), + MulInt64.into(), + AddFloat32.into(), + SubFloat32.into(), + // Monotone in the right argument only. + TextConcatBinary.into(), + // Monotone left, and a declared non-monotone control. + AddDateInterval.into(), + AddTimeInterval.into(), + // Batch 2: remaining ordered-domain arithmetic. + SubInt64.into(), + DivInt64.into(), + SubTimestamp.into(), + SubDate.into(), + AddInterval.into(), + SubInterval.into(), + // Batch 3: the int16 and unsigned arithmetic families, remaining + // date/time arithmetic, and binary date_bin. + AddInt16.into(), + SubInt16.into(), + MulInt16.into(), + DivInt16.into(), + AddUint16.into(), + SubUint16.into(), + MulUint16.into(), + DivUint16.into(), + AddUint32.into(), + SubUint32.into(), + MulUint32.into(), + DivUint32.into(), + AddUint64.into(), + SubUint64.into(), + MulUint64.into(), + DivUint64.into(), + SubTime.into(), + SubTimestampTz.into(), + AddDateTime.into(), + SubDateInterval.into(), + DateBinTimestamp.into(), ] } @@ -1497,6 +1794,75 @@ mod tests { arg0.scalar_type == ReprScalarType::Jsonb && arg1.scalar_type == ReprScalarType::String } + AddInt32(_) | SubInt32(_) | MulInt32(_) | DivInt32(_) => { + arg0.scalar_type == ReprScalarType::Int32 + && arg1.scalar_type == ReprScalarType::Int32 + } + AddInt64(_) | MulInt64(_) | SubInt64(_) | DivInt64(_) => { + arg0.scalar_type == ReprScalarType::Int64 + && arg1.scalar_type == ReprScalarType::Int64 + } + SubTimestamp(_) => { + arg0.scalar_type == ReprScalarType::Timestamp + && arg1.scalar_type == ReprScalarType::Timestamp + } + SubDate(_) => { + arg0.scalar_type == ReprScalarType::Date && arg1.scalar_type == ReprScalarType::Date + } + AddInterval(_) | SubInterval(_) => { + arg0.scalar_type == ReprScalarType::Interval + && arg1.scalar_type == ReprScalarType::Interval + } + AddInt16(_) | SubInt16(_) | MulInt16(_) | DivInt16(_) => { + arg0.scalar_type == ReprScalarType::Int16 + && arg1.scalar_type == ReprScalarType::Int16 + } + AddUint16(_) | SubUint16(_) | MulUint16(_) | DivUint16(_) => { + arg0.scalar_type == ReprScalarType::UInt16 + && arg1.scalar_type == ReprScalarType::UInt16 + } + AddUint32(_) | SubUint32(_) | MulUint32(_) | DivUint32(_) => { + arg0.scalar_type == ReprScalarType::UInt32 + && arg1.scalar_type == ReprScalarType::UInt32 + } + AddUint64(_) | SubUint64(_) | MulUint64(_) | DivUint64(_) => { + arg0.scalar_type == ReprScalarType::UInt64 + && arg1.scalar_type == ReprScalarType::UInt64 + } + SubTime(_) => { + arg0.scalar_type == ReprScalarType::Time && arg1.scalar_type == ReprScalarType::Time + } + SubTimestampTz(_) => { + arg0.scalar_type == ReprScalarType::TimestampTz + && arg1.scalar_type == ReprScalarType::TimestampTz + } + AddDateTime(_) => { + arg0.scalar_type == ReprScalarType::Date && arg1.scalar_type == ReprScalarType::Time + } + SubDateInterval(_) => { + arg0.scalar_type == ReprScalarType::Date + && arg1.scalar_type == ReprScalarType::Interval + } + DateBinTimestamp(_) => { + arg0.scalar_type == ReprScalarType::Interval + && arg1.scalar_type == ReprScalarType::Timestamp + } + AddFloat32(_) | SubFloat32(_) => { + arg0.scalar_type == ReprScalarType::Float32 + && arg1.scalar_type == ReprScalarType::Float32 + } + TextConcat(_) => { + arg0.scalar_type == ReprScalarType::String + && arg1.scalar_type == ReprScalarType::String + } + AddDateInterval(_) => { + arg0.scalar_type == ReprScalarType::Date + && arg1.scalar_type == ReprScalarType::Interval + } + AddTimeInterval(_) => { + arg0.scalar_type == ReprScalarType::Time + && arg1.scalar_type == ReprScalarType::Interval + } _ => false, } } @@ -1892,10 +2258,20 @@ mod tests { // (see the `prop_filter_map`s in `gen_expr_for_relation`), so the // per-run local-reject budget has to be raised well above proptest's // default to let enough cases through. + // An explicit PROPTEST_CASES (already parsed into the default config) + // wins, for long local or nightly runs. The generator rejects at a + // roughly fixed rate per case, so the reject budget scales with the + // case count. + let default = ProptestConfig::default(); + let cases = if std::env::var_os("PROPTEST_CASES").is_some() { + default.cases + } else { + 2048 + }; let config = ProptestConfig { - cases: 2048, - max_local_rejects: 1 << 20, - ..ProptestConfig::default() + cases, + max_local_rejects: cases.saturating_mul(512), + ..default }; proptest!(config, |(data in gen_range_expr_data())| { check(data)?; diff --git a/src/repr/src/scalar.rs b/src/repr/src/scalar.rs index f62069e26a17c..deb382e96e9ce 100644 --- a/src/repr/src/scalar.rs +++ b/src/repr/src/scalar.rs @@ -4053,6 +4053,12 @@ impl SqlScalarType { Datum::Float32(OrderedFloat(f32::MAX)), Datum::Float32(OrderedFloat(f32::EPSILON)), Datum::Float32(OrderedFloat(f32::NAN)), + // NOTE: -NaN and -0.0 have distinct bit patterns from NaN and + // 0.0 but compare equal under `OrderedFloat`. Orderings that + // look at the representation (e.g. arrow's total order, where + // -NaN < -Infinity) can disagree with `OrderedFloat` on them. + Datum::Float32(OrderedFloat(-f32::NAN)), + Datum::Float32(OrderedFloat(-0.0)), Datum::Float32(OrderedFloat(f32::INFINITY)), Datum::Float32(OrderedFloat(f32::NEG_INFINITY)), ]) @@ -4067,6 +4073,9 @@ impl SqlScalarType { Datum::Float64(OrderedFloat(f64::MAX)), Datum::Float64(OrderedFloat(f64::EPSILON)), Datum::Float64(OrderedFloat(f64::NAN)), + // See the FLOAT32 note on -NaN and -0.0. + Datum::Float64(OrderedFloat(-f64::NAN)), + Datum::Float64(OrderedFloat(-0.0)), Datum::Float64(OrderedFloat(f64::INFINITY)), Datum::Float64(OrderedFloat(f64::NEG_INFINITY)), ]) @@ -4103,6 +4112,9 @@ impl SqlScalarType { Row::pack_slice(&[ Datum::Time(NaiveTime::from_hms_micro_opt(0, 0, 0, 0).unwrap()), Datum::Time(NaiveTime::from_hms_micro_opt(23, 59, 59, 999_999).unwrap()), + // Leap second: chrono represents it as a fractional part of + // one second or more. + Datum::Time(NaiveTime::from_hms_micro_opt(23, 59, 59, 1_999_999).unwrap()), ]) }); static TIMESTAMP: LazyLock = LazyLock::new(|| { @@ -4225,6 +4237,13 @@ impl SqlScalarType { Datum::String("."), Datum::String("2015-09-18T23:56:04.123Z"), Datum::String(&"x".repeat(100)), + // Persist stats truncate string bounds to 100 bytes: cover a + // string past that limit, one whose truncated upper bound + // cannot be incremented (every char is char::MAX), and one + // with a multibyte char straddling the truncation boundary. + Datum::String(&"x".repeat(101)), + Datum::String(&"\u{10FFFF}".repeat(101)), + Datum::String(&format!("{}\u{1F600}", "x".repeat(99))), // Valid timezone. Datum::String("JAPAN"), Datum::String("1,2,3"), @@ -4267,8 +4286,31 @@ impl SqlScalarType { // JSON doesn't support NaN or Infinite numbers. !(n.0.is_nan() || n.0.is_infinite()) })); - // TODO: Add List, Map. - Row::pack_slice(&datums) + let mut row = Row::default(); + let mut packer = row.packer(); + for datum in datums { + packer.push(datum); + } + // Maps, including ones with disjoint key sets. Persist keeps + // per-key statistics for JSON maps, so a collection mixing maps + // where a key is present in one and absent in another exercises + // the absent-key handling in stats and their consumers. + packer.push_dict([("x", Datum::String("a"))]); + packer.push_dict([("y", Datum::String("b"))]); + packer.push_dict([("x", Datum::True), ("y", Datum::JsonNull)]); + packer.push_dict(std::iter::empty::<(&str, Datum)>()); + // JSON map keys are not truncated in persist stats, unlike SQL + // string columns, so cover one past the string truncation limit. + let long_key = "k".repeat(101); + packer.push_dict([(long_key.as_str(), Datum::True)]); + packer.push_dict_with(|packer| { + packer.push(Datum::String("nested")); + packer.push_dict([("x", Datum::String("a"))]); + }); + // Lists, including a heterogeneous one. + packer.push_list([Datum::True, Datum::JsonNull, Datum::String("a")]); + packer.push_list(std::iter::empty::()); + row }); static UUID: LazyLock = LazyLock::new(|| { Row::pack_slice(&[ diff --git a/src/storage-operators/proptest-regressions/persist_source.txt b/src/storage-operators/proptest-regressions/persist_source.txt index fa00449dfd034..764b5eed6d590 100644 --- a/src/storage-operators/proptest-regressions/persist_source.txt +++ b/src/storage-operators/proptest-regressions/persist_source.txt @@ -5,3 +5,5 @@ # It is recommended to check this file in to source control so that # everyone who runs the test benefits from these saved cases. cc 30de45b312c033e03a3b6d1ade21cbefa00db4158eb3934456597e55e387361a # shrinks to rows = [Row{[Numeric(1)]}, Row{[Numeric(0.25)]}, Row{[Numeric(1.5)]}], predicate = CallBinary(Gte(Gte), CallBinary(MulFloat32(MulFloat32), CallBinary(AddFloat32(AddFloat32), Literal(Ok(Row{[Float32(0.0)]}), ReprColumnType { scalar_type: Float32, nullable: false }), CallUnary(CastNumericToFloat32(CastNumericToFloat32), CallBinary(RoundNumeric(RoundNumericBinary), Column(0), Literal(Ok(Row{[Int32(24699)]}), ReprColumnType { scalar_type: Int32, nullable: false })))), Literal(Ok(Row{[Float32(0.0)]}), ReprColumnType { scalar_type: Float32, nullable: false })), Literal(Ok(Row{[Float32(0.0087531805)]}), ReprColumnType { scalar_type: Float32, nullable: false })) +cc 707390daf9721b02c362f9a5fe0043732b86e262553091eeb84eb9462674166a # shrinks to rows = [SourceData(Ok(Row{[Numeric(0), Float32(NaN), Float64(0.0), String(""), True, True, True, Timestamp(1970-01-01T00:00:00), MzTimestamp(0)]})), SourceData(Ok(Row{[Numeric(0), Float32(0.0), Float64(0.0), String(""), True, True, True, Timestamp(1970-01-01T00:00:00), MzTimestamp(0)]}))], predicate = CallBinary(Gte(Gte), Column(1), Literal(Ok(Row{[Float32(0.0)]}), ReprColumnType { scalar_type: Float32, nullable: false })), eval_time = 1, until = Some(5) +cc 2346210a91590a7e75f274cae303f9c7fc12fd6b34e216dd47ee31802da674b6 # shrinks to rows = [SourceData(Ok(Row{[Numeric(0), Float32(0.0), Float64(-1.0), String(""), True, True, True, Timestamp(1970-01-01T00:00:00), MzTimestamp(0)]})), SourceData(Ok(Row{[Numeric(0), Float32(0.0), Float64(NaN), String(""), True, True, True, Timestamp(1970-01-01T00:00:00), MzTimestamp(0)]})), SourceData(Ok(Row{[Numeric(0), Float32(0.0), Float64(NaN), String(""), True, True, True, Timestamp(1970-01-01T00:00:00), MzTimestamp(0)]}))], predicate = CallUnary(Not(Not), CallBinary(Gte(Gte), CallBinary(MulFloat64(MulFloat64), Column(2), Literal(Ok(Row{[Float64(0.0)]}), ReprColumnType { scalar_type: Float64, nullable: false })), Literal(Ok(Row{[Float64(1.0)]}), ReprColumnType { scalar_type: Float64, nullable: false }))), eval_time = 1, until = Some(5) diff --git a/src/storage-operators/src/persist_source.rs b/src/storage-operators/src/persist_source.rs index 700c088447da4..f08e0cc28df49 100644 --- a/src/storage-operators/src/persist_source.rs +++ b/src/storage-operators/src/persist_source.rs @@ -1534,18 +1534,21 @@ mod tests { /// column stat range that fails to contain a real value). See /// database-issues#9656 / PER-50. mod filter_pushdown_audit { + use mz_expr::func::variadic::{And, Or}; use mz_expr::func::{ - AddFloat32, CastNumericToFloat32, CastNumericToMzTimestamp, Eq, Gt, Gte, Lt, Lte, - MulFloat32, RoundNumericBinary, + AddFloat32, AddTimestampInterval, CastNumericToFloat32, CastNumericToMzTimestamp, Eq, + Gt, Gte, IsNull, JsonbGetString, JsonbGetStringStringify, Lt, Lte, MulFloat32, + MulFloat64, Not, RoundNumericBinary, TryParseMonotonicIso8601Timestamp, }; use mz_expr::{BinaryFunc, MapFilterProject, MirScalarExpr, UnaryFunc}; use mz_ore::metrics::MetricsRegistry; use mz_persist_types::part::PartBuilder; use mz_persist_types::stats::{PartStats, PartStatsMetrics}; + use mz_repr::adt::interval::Interval; use mz_repr::adt::numeric::Numeric; use mz_repr::{Diff, ReprScalarType, SqlScalarType}; use proptest::prelude::*; - use proptest::sample::select; + use proptest::sample::{Index, select}; use super::*; @@ -1572,10 +1575,10 @@ mod tests { /// Compute the real production `PartStats` from a set of rows, the same /// way the storage read path does. - fn build_part_stats(desc: &RelationDesc, rows: &[Row]) -> PartStats { + fn build_part_stats(desc: &RelationDesc, rows: &[SourceData]) -> PartStats { let mut builder = PartBuilder::new(desc, &UnitSchema); for row in rows { - builder.push(&SourceData(Ok(row.clone())), &(), 1u64, 1i64); + builder.push(row, &(), 1u64, 1i64); } let part = builder.finish(); PartStats::new::(&part, desc).expect("stats") @@ -1720,7 +1723,8 @@ mod tests { .into_plan() .expect("into_plan"); - let part_stats = build_part_stats(&desc, &rows); + let source_rows: Vec<_> = rows.iter().map(|r| SourceData(Ok(r.clone()))).collect(); + let part_stats = build_part_stats(&desc, &source_rows); let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); let stats = RelationPartStats::new("test", &metrics, &desc, &part_stats); @@ -1756,7 +1760,8 @@ mod tests { rows={rows:?}\nplan={plan:?}", ); - let part_stats = build_part_stats(desc, rows); + let source_rows: Vec<_> = rows.iter().map(|r| SourceData(Ok(r.clone()))).collect(); + let part_stats = build_part_stats(desc, &source_rows); let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); let stats = RelationPartStats::new("test", &metrics, desc, &part_stats); let decision = filter_result(desc, ResultSpec::anything(), stats, &plan); @@ -1866,5 +1871,626 @@ mod tests { }, ); } + + // Wide-schema variant: multiple column types populated from + // `interesting_datums`, a predicate vocabulary that reaches the + // interpreter's special cases (jsonb map specs and their unions, + // `TryParseMonotonicIso8601Timestamp`, dynamically-monotone + // timestamp+interval, the infinity guard on float multiplication), + // Err rows, and real mz_now bounds instead of an unconstrained time + // range. + + const NUM: usize = 0; + const F32: usize = 1; + const F64: usize = 2; + const STR: usize = 3; + const J1: usize = 4; + const J2: usize = 5; + const BOOL: usize = 6; + const TS: usize = 7; + const MZTS: usize = 8; + const WIDE_ARITY: usize = 9; + + fn wide_scalar_type(col: usize) -> SqlScalarType { + match col { + NUM => SqlScalarType::Numeric { max_scale: None }, + F32 => SqlScalarType::Float32, + F64 => SqlScalarType::Float64, + STR => SqlScalarType::String, + J1 | J2 => SqlScalarType::Jsonb, + BOOL => SqlScalarType::Bool, + TS => SqlScalarType::Timestamp { precision: None }, + MZTS => SqlScalarType::MzTimestamp, + _ => unreachable!("no such column"), + } + } + + fn wide_repr_type(col: usize) -> ReprScalarType { + match col { + NUM => ReprScalarType::Numeric, + F32 => ReprScalarType::Float32, + F64 => ReprScalarType::Float64, + STR => ReprScalarType::String, + J1 | J2 => ReprScalarType::Jsonb, + BOOL => ReprScalarType::Bool, + TS => ReprScalarType::Timestamp, + MZTS => ReprScalarType::MzTimestamp, + _ => unreachable!("no such column"), + } + } + + fn wide_desc() -> RelationDesc { + let mut builder = RelationDesc::builder(); + for col in 0..WIDE_ARITY { + // The bool column stays non-nullable so an all-Err part + // exercises the fabricated default bounds a non-nullable + // column gets when no Ok row provides a value. + let nullable = col != BOOL; + builder = builder + .with_column(format!("c{col}"), wide_scalar_type(col).nullable(nullable)); + } + builder.finish() + } + + fn wide_pool(col: usize) -> Vec> { + let mut pool: Vec<_> = wide_scalar_type(col).interesting_datums().collect(); + if col != BOOL { + pool.push(Datum::Null); + } + pool + } + + fn arb_wide_rows() -> impl Strategy> { + let pools: Vec>> = (0..WIDE_ARITY).map(wide_pool).collect(); + let ok_row = prop::collection::vec(any::(), WIDE_ARITY).prop_map(move |picks| { + let datums = picks + .iter() + .zip(&pools) + .map(|(pick, pool)| pool[pick.index(pool.len())]); + SourceData(Ok(Row::pack(datums))) + }); + let err_row = Just(SourceData(Err(DataflowError::from( + EvalError::DivisionByZero, + )))); + let row = prop_oneof![9 => ok_row, 1 => err_row]; + prop::collection::vec(row, 2..8) + } + + fn lit(datum: Datum<'static>, typ: ReprScalarType) -> MirScalarExpr { + if datum.is_null() { + MirScalarExpr::literal_null(typ) + } else { + MirScalarExpr::literal_ok(datum, typ) + } + } + + fn is_null(expr: MirScalarExpr) -> MirScalarExpr { + MirScalarExpr::CallUnary { + func: UnaryFunc::IsNull(IsNull), + expr: Box::new(expr), + } + } + + fn not(expr: MirScalarExpr) -> MirScalarExpr { + MirScalarExpr::CallUnary { + func: UnaryFunc::Not(Not), + expr: Box::new(expr), + } + } + + fn binary(func: BinaryFunc, a: MirScalarExpr, b: MirScalarExpr) -> MirScalarExpr { + MirScalarExpr::CallBinary { + func, + expr1: Box::new(a), + expr2: Box::new(b), + } + } + + /// `col lit`, with the literal drawn from the same interesting + /// pool as the row values, so poison values show up on both sides. + fn arb_cmp_col_lit() -> impl Strategy { + (0..WIDE_ARITY, any::(), comparison_funcs()).prop_map(|(col, pick, cmp)| { + let pool = wide_pool(col); + let datum = pool[pick.index(pool.len())]; + binary( + cmp, + MirScalarExpr::column(col), + lit(datum, wide_repr_type(col)), + ) + }) + } + + fn arb_is_null_pred() -> impl Strategy { + (0..WIDE_ARITY, any::()).prop_map(|(col, negate)| { + let expr = is_null(MirScalarExpr::column(col)); + if negate { not(expr) } else { expr } + }) + } + + fn jsonb_keys() -> impl Strategy { + select(vec!["x", "y", "nested", "absent"]) + } + + fn jsonb_get(expr: MirScalarExpr, key: &'static str, stringify: bool) -> MirScalarExpr { + let func = if stringify { + BinaryFunc::JsonbGetStringStringify(JsonbGetStringStringify) + } else { + BinaryFunc::JsonbGetString(JsonbGetString) + }; + binary( + func, + expr, + MirScalarExpr::literal_ok(Datum::String(key), ReprScalarType::String), + ) + } + + /// `(jN -> 'key') IS NULL` or `(jN ->> 'key') = 'a'`, the shapes that + /// consume the Nested specs built from real jsonb map stats. + fn arb_jsonb_pred() -> impl Strategy { + ( + select(vec![J1, J2]), + jsonb_keys(), + any::(), + any::(), + ) + .prop_map(|(col, key, stringify, wrap_eq)| { + let get = jsonb_get(MirScalarExpr::column(col), key, stringify); + if wrap_eq { + let typ = if stringify { + ReprScalarType::String + } else { + ReprScalarType::Jsonb + }; + binary(BinaryFunc::Eq(Eq), get, lit(Datum::String("a"), typ)) + } else { + is_null(get) + } + }) + } + + /// `((CASE WHEN THEN j1 ELSE j2 END) ->> 'key') IS NULL`, the + /// PER-6 shape: unioning the two columns' Nested specs. + fn arb_case_jsonb_pred() -> impl Strategy { + (any::(), jsonb_keys(), any::()).prop_map( + |(cond_is_col, key, stringify)| { + let cond = if cond_is_col { + MirScalarExpr::column(BOOL) + } else { + is_null(MirScalarExpr::column(STR)) + }; + let case = MirScalarExpr::If { + cond: Box::new(cond), + then: Box::new(MirScalarExpr::column(J1)), + els: Box::new(MirScalarExpr::column(J2)), + }; + is_null(jsonb_get(case, key, stringify)) + }, + ) + } + + /// `try_parse_monotonic_iso8601_timestamp(c_str) `, the one + /// SpecialUnary implementation in the interpreter. + fn arb_iso_parse_pred() -> impl Strategy { + (comparison_funcs(), any::(), any::()).prop_map( + |(cmp, pick, wrap_null)| { + let parse = MirScalarExpr::CallUnary { + func: UnaryFunc::TryParseMonotonicIso8601Timestamp( + TryParseMonotonicIso8601Timestamp, + ), + expr: Box::new(MirScalarExpr::column(STR)), + }; + if wrap_null { + is_null(parse) + } else { + let pool: Vec<_> = SqlScalarType::Timestamp { precision: None } + .interesting_datums() + .collect(); + let datum = pool[pick.index(pool.len())]; + binary(cmp, parse, lit(datum, ReprScalarType::Timestamp)) + } + }, + ) + } + + /// `(c_ts + ) `, the DynamicMonotone handler: + /// day-only intervals are treated as monotone, month-bearing ones must + /// stay conservative. + fn arb_ts_interval_pred() -> impl Strategy { + let intervals = select(vec![ + Interval::new(0, 2, 0), + Interval::new(0, 0, 3_600_000_000), + Interval::new(1, 0, 0), + Interval::new(-1, 0, 0), + ]); + (comparison_funcs(), intervals, any::()).prop_map(|(cmp, iv, pick)| { + let add = binary( + BinaryFunc::AddTimestampInterval(AddTimestampInterval), + MirScalarExpr::column(TS), + lit(Datum::Interval(iv), ReprScalarType::Interval), + ); + let pool: Vec<_> = SqlScalarType::Timestamp { precision: None } + .interesting_datums() + .collect(); + let datum = pool[pick.index(pool.len())]; + binary(cmp, add, lit(datum, ReprScalarType::Timestamp)) + }) + } + + /// `(c_f64 * ) `, aimed at the interpreter's + /// infinity guard: multiplication is monotone but not + /// infinity-monotone. + fn arb_float_mul_pred() -> impl Strategy { + let consts = || select(vec![0.0f64, 1.0, -1.0, 1e300, -1e300, f64::INFINITY]); + (comparison_funcs(), consts(), consts()).prop_map(|(cmp, a, c)| { + let mul = binary( + BinaryFunc::MulFloat64(MulFloat64), + MirScalarExpr::column(F64), + lit(Datum::from(a), ReprScalarType::Float64), + ); + binary(cmp, mul, lit(Datum::from(c), ReprScalarType::Float64)) + }) + } + + /// `mz_now() `, compiled by `into_plan` into + /// the temporal lower/upper bounds that `filter_result` checks against + /// the part's time range. + fn arb_temporal_pred() -> impl Strategy { + let cmps = select(vec![ + BinaryFunc::Lte(Lte), + BinaryFunc::Lt(Lt), + BinaryFunc::Gte(Gte), + BinaryFunc::Gt(Gt), + ]); + (cmps, any::(), any::()).prop_map(|(cmp, use_col, pick)| { + let mz_now = MirScalarExpr::CallUnmaterializable(UnmaterializableFunc::MzNow); + let rhs = if use_col { + MirScalarExpr::column(MZTS) + } else { + let pool = wide_pool(MZTS); + lit(pool[pick.index(pool.len())], ReprScalarType::MzTimestamp) + }; + binary(cmp, mz_now, rhs) + }) + } + + fn arb_wide_predicate() -> impl Strategy { + let leaf = prop_oneof![ + arb_cmp_col_lit(), + arb_is_null_pred(), + arb_jsonb_pred(), + arb_case_jsonb_pred(), + arb_iso_parse_pred(), + arb_ts_interval_pred(), + arb_float_mul_pred(), + arb_temporal_pred(), + // The numeric shapes from the narrow test, aimed at the + // fallible-interior mechanisms; both reference column 0. + ( + select(vec![0i32, 2, -5, 24699]), + f32_consts(), + f32_consts(), + f32_consts(), + comparison_funcs() + ) + .prop_map(|(s, a, b, c, cmp)| float_arith_predicate(s, a, b, c, cmp)), + (select(vec![0u64, 1, 2, 100, u64::MAX]), comparison_funcs()) + .prop_map(|(ts, cmp)| cast_mz_timestamp_predicate(ts, cmp)), + ] + .boxed(); + prop_oneof![ + 3 => leaf.clone(), + 1 => (leaf.clone(), leaf.clone(), any::()).prop_map(|(a, b, is_and)| { + let func = if is_and { And.into() } else { Or.into() }; + MirScalarExpr::CallVariadic { func, exprs: vec![a, b] } + }), + 1 => leaf.prop_map(not), + ] + } + + /// The zero-column count(*) path: when the read desc projects away + /// every column and each row is known to pass, `filter_result` + /// replaces the part with a synthesized single-row KV instead of + /// keeping or discarding it. Errors and filters that can skip rows + /// must suppress the substitution. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // too slow + fn zero_column_relation_replace_with() { + let desc = RelationDesc::empty(); + let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); + let ok_rows = vec![ + SourceData(Ok(Row::default())), + SourceData(Ok(Row::default())), + ]; + + // No predicates, no errors: every row passes, so the part is + // replaced with the synthesized KV. + let plan = MapFilterProject::new(0).into_plan().expect("into_plan"); + let part_stats = build_part_stats(&desc, &ok_rows); + let stats = RelationPartStats::new("test", &metrics, &desc, &part_stats); + let decision = filter_result(&desc, ResultSpec::anything(), stats, &plan); + assert!( + matches!(decision, FilterResult::ReplaceWith { .. }), + "expected ReplaceWith, got {decision:?}", + ); + + // An error row must disable the substitution: the part has to be + // fetched so the error surfaces. + let mixed_rows = vec![ + SourceData(Ok(Row::default())), + SourceData(Err(DataflowError::from(EvalError::DivisionByZero))), + ]; + let part_stats = build_part_stats(&desc, &mixed_rows); + let stats = RelationPartStats::new("test", &metrics, &desc, &part_stats); + let decision = filter_result(&desc, ResultSpec::anything(), stats, &plan); + assert!( + matches!(decision, FilterResult::Keep), + "expected Keep, got {decision:?}", + ); + + // A constant-false filter never keeps anything: plain Discard. + let plan = MapFilterProject::new(0) + .filter(std::iter::once(MirScalarExpr::literal_ok( + Datum::False, + ReprScalarType::Bool, + ))) + .into_plan() + .expect("into_plan"); + let part_stats = build_part_stats(&desc, &ok_rows); + let stats = RelationPartStats::new("test", &metrics, &desc, &part_stats); + let decision = filter_result(&desc, ResultSpec::anything(), stats, &plan); + assert!( + matches!(decision, FilterResult::Discard), + "expected Discard, got {decision:?}", + ); + } + + /// Schema drift between the stats and the read desc must degrade to + /// "no stats", never to a narrower spec. + /// + /// Two real shapes: a column appended by `ALTER TABLE ... ADD COLUMN` + /// after the part was written (present in the read desc, absent from + /// the stats), and demand pushdown projecting the read desc down to a + /// subset of the written columns (stats carry extra columns). + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // too slow + fn schema_drift_degrades_to_no_stats() { + let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); + + // Part written before ALTER TABLE ... ADD COLUMN b. + let write_desc = RelationDesc::builder() + .with_column("a", SqlScalarType::Int32.nullable(false)) + .finish(); + let rows = vec![SourceData(Ok(Row::pack_slice(&[Datum::Int32(1)])))]; + let part_stats = build_part_stats(&write_desc, &rows); + + let read_desc = RelationDesc::builder() + .with_column("a", SqlScalarType::Int32.nullable(false)) + .with_column("b", SqlScalarType::Float64.nullable(true)) + .finish(); + let stats = RelationPartStats::new("test", &metrics, &read_desc, &part_stats); + // Old rows read the new column as null, so `b IS NULL` matches + // them and the part must be kept. + let plan = MapFilterProject::new(2) + .filter(std::iter::once(is_null(MirScalarExpr::column(1)))) + .into_plan() + .expect("into_plan"); + let decision = filter_result(&read_desc, ResultSpec::anything(), stats, &plan); + assert!( + !matches!(decision, FilterResult::Discard), + "part written before ADD COLUMN was discarded: {decision:?}", + ); + + // Demand pushdown: the read desc is a projection of the written + // schema. The surviving column's stats must still line up with it + // by name, so a matching filter keeps the part. + let write_desc = RelationDesc::builder() + .with_column("a", SqlScalarType::Int32.nullable(false)) + .with_column("b", SqlScalarType::Float64.nullable(true)) + .finish(); + let rows = vec![SourceData(Ok(Row::pack_slice(&[ + Datum::Int32(1), + Datum::from(5.0f64), + ])))]; + let part_stats = build_part_stats(&write_desc, &rows); + + let read_desc = RelationDesc::builder() + .with_column("b", SqlScalarType::Float64.nullable(true)) + .finish(); + let stats = RelationPartStats::new("test", &metrics, &read_desc, &part_stats); + let plan = MapFilterProject::new(1) + .filter(std::iter::once(binary( + BinaryFunc::Eq(Eq), + MirScalarExpr::column(0), + lit(Datum::from(5.0f64), ReprScalarType::Float64), + ))) + .into_plan() + .expect("into_plan"); + let decision = filter_result(&read_desc, ResultSpec::anything(), stats, &plan); + assert!( + !matches!(decision, FilterResult::Discard), + "projected read desc discarded a matching part: {decision:?}", + ); + } + + /// Ground truth, mirroring [`PendingWork::do_work`]: a part yields + /// output if any Err row survives `until`, or if the MFP applied to + /// any Ok row at its effective time produces anything at all. The + /// runtime audit fires on any result from `evaluate`, before the + /// additional `until` filtering of the produced rows, so this must + /// not post-filter either. + fn part_yields_output( + plan: &MfpPlan, + rows: &[SourceData], + eval_time: Timestamp, + until: &Antichain, + ) -> bool { + if until.less_equal(&eval_time) { + return false; + } + let arena = RowArena::new(); + let mut row_builder = Row::default(); + for source_data in rows { + match &source_data.0 { + Err(_) => return true, + Ok(row) => { + let mut datums: Vec = row.iter().collect(); + let mut results = plan.evaluate::( + &mut datums, + &arena, + eval_time, + Diff::from(1), + |time| !until.less_equal(time), + &mut row_builder, + ); + if results.next().is_some() { + return true; + } + } + } + } + false + } + + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // too slow, and decNumber FFI is unsupported + fn wide_filter_result_never_discards_matching_part() { + fn check( + rows: Vec, + predicate: MirScalarExpr, + eval_time: u64, + until: Option, + ) -> Result<(), TestCaseError> { + let desc = wide_desc(); + // Predicate shapes that use mz_now in a way the temporal + // filter machinery does not support fail to plan; there is + // nothing to check for those. + let Ok(plan) = MapFilterProject::new(desc.arity()) + .filter(std::iter::once(predicate)) + .into_plan() + else { + return Ok(()); + }; + let eval_time = Timestamp::from(eval_time); + let until = + until.map_or_else(Antichain::new, |t| Antichain::from_elem(Timestamp::from(t))); + + let part_stats = build_part_stats(&desc, &rows); + let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); + let stats = RelationPartStats::new("test", &metrics, &desc, &part_stats); + + // Mirror the read path: mz_now is bounded by the part's + // frontier and the dataflow's until, both inclusive, with an + // empty until standing in for MAX. A frontier past the until + // is discarded before stats are consulted. + let upper = until.as_option().copied().unwrap_or(Timestamp::MAX); + if eval_time > upper { + return Ok(()); + } + let time_range = ResultSpec::value_between( + Datum::MzTimestamp(eval_time), + Datum::MzTimestamp(upper), + ); + let decision = filter_result(&desc, time_range, stats, &plan); + + if part_yields_output(&plan, &rows, eval_time, &until) { + prop_assert!( + !matches!(decision, FilterResult::Discard), + "filter pushdown discarded a part whose MFP yields output on a real \ + row (wrongly-skipped part; the runtime audit would panic).\n\ + rows={rows:?}\nplan={plan:?}\neval_time={eval_time}\nuntil={until:?}", + ); + } + Ok(()) + } + + // The vocabulary is wide (9 columns, 10 predicate shapes), so a + // specific poison-value-plus-predicate coincidence is rare per + // case. The default 256 cases demonstrably miss known bugs; 4096 + // still runs in a couple of seconds because each case is cheap. + // An explicit PROPTEST_CASES (already parsed into the default + // config) wins, for long local or nightly runs. + let default = ProptestConfig::default(); + let cases = if std::env::var_os("PROPTEST_CASES").is_some() { + default.cases + } else { + 4096 + }; + let config = ProptestConfig { cases, ..default }; + proptest!(config, |( + rows in arb_wide_rows(), + predicate in arb_wide_predicate(), + eval_time in select(vec![1u64, 5]), + until in select(vec![None, Some(1u64), Some(5), Some(8), Some(100)]), + )| { + check(rows, predicate, eval_time, until)?; + }); + } + + /// Filter decisions are made per part: rows split across several + /// parts get an independent decision per part, and a part whose own + /// rows yield output must never be discarded, regardless of what the + /// sibling parts contain (e.g. a poison value in one part must not + /// affect another part's decision, and vice versa). + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // too slow, and decNumber FFI is unsupported + fn multi_part_decisions_are_independent() { + fn check( + parts: Vec>, + predicate: MirScalarExpr, + eval_time: u64, + until: Option, + ) -> Result<(), TestCaseError> { + let desc = wide_desc(); + let Ok(plan) = MapFilterProject::new(desc.arity()) + .filter(std::iter::once(predicate)) + .into_plan() + else { + return Ok(()); + }; + let eval_time = Timestamp::from(eval_time); + let until = + until.map_or_else(Antichain::new, |t| Antichain::from_elem(Timestamp::from(t))); + let upper = until.as_option().copied().unwrap_or(Timestamp::MAX); + if eval_time > upper { + return Ok(()); + } + let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); + + for rows in &parts { + let part_stats = build_part_stats(&desc, rows); + let stats = RelationPartStats::new("test", &metrics, &desc, &part_stats); + let time_range = ResultSpec::value_between( + Datum::MzTimestamp(eval_time), + Datum::MzTimestamp(upper), + ); + let decision = filter_result(&desc, time_range, stats, &plan); + if part_yields_output(&plan, rows, eval_time, &until) { + prop_assert!( + !matches!(decision, FilterResult::Discard), + "filter pushdown discarded a part whose MFP yields output on a \ + real row.\nrows={rows:?}\nplan={plan:?}\neval_time={eval_time}\n\ + until={until:?}", + ); + } + } + Ok(()) + } + + let default = ProptestConfig::default(); + let cases = if std::env::var_os("PROPTEST_CASES").is_some() { + default.cases + } else { + 1024 + }; + let config = ProptestConfig { cases, ..default }; + proptest!(config, |( + parts in prop::collection::vec(arb_wide_rows(), 2..4), + predicate in arb_wide_predicate(), + eval_time in select(vec![1u64, 5]), + until in select(vec![None, Some(5u64), Some(100)]), + )| { + check(parts, predicate, eval_time, until)?; + }); + } } } diff --git a/src/storage-types/fuzz/Cargo.toml b/src/storage-types/fuzz/Cargo.toml index 25574d558048e..d5f1d7c630606 100644 --- a/src/storage-types/fuzz/Cargo.toml +++ b/src/storage-types/fuzz/Cargo.toml @@ -20,7 +20,11 @@ edition = "2021" cargo-fuzz = true [dependencies] +arbitrary = { version = "1", features = ["derive"] } libfuzzer-sys = "0.4" +mz-expr = { path = "../../expr" } +mz-ore = { path = "../../ore" } +mz-persist-types = { path = "../../persist-types" } mz-storage-types = { path = "..", features = ["proptest"] } mz-proto = { path = "../../proto" } mz-repr = { path = "../../repr", features = ["proptest"] } @@ -41,6 +45,13 @@ test = false doc = false bench = false +[[bin]] +name = "pushdown_soundness" +path = "fuzz_targets/pushdown_soundness.rs" +test = false +doc = false +bench = false + [[bin]] name = "source_data_proto_roundtrip" path = "fuzz_targets/source_data_proto_roundtrip.rs" diff --git a/src/storage-types/fuzz/fuzz_targets/pushdown_soundness.rs b/src/storage-types/fuzz/fuzz_targets/pushdown_soundness.rs new file mode 100644 index 0000000000000..b38ebf3faaea4 --- /dev/null +++ b/src/storage-types/fuzz/fuzz_targets/pushdown_soundness.rs @@ -0,0 +1,337 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file at the root of this repository. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Coverage-guided check of the persist filter pushdown soundness property: +//! a part whose rows produce output under an MFP must never be reported as +//! irrelevant by [`RelationPartStats::may_match_mfp`], the pushdown entry +//! point of the peek path. +//! +//! The whole pipeline runs on real production code: rows are packed into a +//! persist part, the part's column statistics are computed by the write +//! path, and the interpreter consumes them exactly as pushdown does. The +//! fuzzer's value over the proptest harnesses in `mz_storage_operators`'s +//! `filter_pushdown_audit` module is raw bit-pattern access: float columns +//! take arbitrary `u64` bit patterns (every NaN payload and sign, subnormals) +//! and strings take arbitrary bytes, while coverage feedback steers the +//! search toward rarely-taken stats and interpreter branches. + +#![no_main] + +use arbitrary::Arbitrary; +use libfuzzer_sys::fuzz_target; +use mz_expr::func::variadic::{And, Or}; +use mz_expr::func::{ + AddFloat64, Eq, Gt, Gte, IsNull, JsonbGetString, JsonbGetStringStringify, Lt, Lte, MulFloat64, + Not, +}; +use mz_expr::{BinaryFunc, MapFilterProject, MirScalarExpr, ResultSpec, UnaryFunc}; +use mz_ore::metrics::MetricsRegistry; +use mz_persist_types::codec_impls::UnitSchema; +use mz_persist_types::part::PartBuilder; +use mz_persist_types::stats::{PartStats, PartStatsMetrics}; +use mz_repr::adt::numeric::Numeric; +use mz_repr::{ + Datum, Diff, RelationDesc, ReprScalarType, Row, RowArena, SqlScalarType, Timestamp, +}; +use mz_storage_types::errors::DataflowError; +use mz_storage_types::sources::SourceData; +use mz_storage_types::stats::RelationPartStats; + +const NUM: usize = 0; +const F64: usize = 1; +const STR: usize = 2; +const JSON: usize = 3; +const BOOL: usize = 4; +const ARITY: usize = 5; + +#[derive(Arbitrary, Debug)] +struct FuzzRow { + num: FuzzDatum, + f64_bits: FuzzDatum, + string: FuzzDatum, + json: FuzzDatum, + bool_null: FuzzDatum, +} + +/// A datum source: raw bits for coverage-guided exploration, or an index into +/// the column type's `interesting_datums` pool for the known poison values. +#[derive(Arbitrary, Debug)] +enum FuzzDatum { + Null, + Bits(u64), + Bytes([u8; 8]), + Pool(u8), +} + +#[derive(Arbitrary, Debug)] +enum Cmp { + Lt, + Lte, + Gt, + Gte, + Eq, +} + +#[derive(Arbitrary, Debug)] +enum Pred { + /// `col lit`, the literal drawn like a row value. + CmpColLit { col: u8, cmp: Cmp, lit: FuzzDatum }, + IsNull { col: u8, negate: bool }, + /// `(json ->(>) 'key') IS NULL` over the Nested specs from real map stats. + JsonbKey { key: u8, stringify: bool }, + /// `(c_f64 + a) * b c`, aimed at the infinity guard and NaN + /// arithmetic, with fuzzer-chosen bit patterns. + FloatArith { cmp: Cmp, a: u64, b: u64, c: u64 }, + And(Box, Box), + Or(Box, Box), + Not(Box), +} + +#[derive(Arbitrary, Debug)] +struct Input { + rows: Vec, + /// Bitmask of error rows appended to the part. + err_rows: u8, + pred: Pred, +} + +fn schema() -> RelationDesc { + RelationDesc::builder() + .with_column("c_num", SqlScalarType::Numeric { max_scale: None }.nullable(true)) + .with_column("c_f64", SqlScalarType::Float64.nullable(true)) + .with_column("c_str", SqlScalarType::String.nullable(true)) + .with_column("c_json", SqlScalarType::Jsonb.nullable(true)) + .with_column("c_bool", SqlScalarType::Bool.nullable(true)) + .finish() +} + +fn col_type(col: usize) -> SqlScalarType { + match col { + NUM => SqlScalarType::Numeric { max_scale: None }, + F64 => SqlScalarType::Float64, + STR => SqlScalarType::String, + JSON => SqlScalarType::Jsonb, + BOOL => SqlScalarType::Bool, + _ => unreachable!(), + } +} + +fn repr_type(col: usize) -> ReprScalarType { + match col { + NUM => ReprScalarType::Numeric, + F64 => ReprScalarType::Float64, + STR => ReprScalarType::String, + JSON => ReprScalarType::Jsonb, + BOOL => ReprScalarType::Bool, + _ => unreachable!(), + } +} + +/// Materialize a datum for `col` into the packer. +fn push_datum(packer: &mut mz_repr::RowPacker, col: usize, d: &FuzzDatum) { + match d { + FuzzDatum::Null => packer.push(Datum::Null), + FuzzDatum::Bits(bits) => match col { + NUM => packer.push(Datum::from(Numeric::from(f64::from_bits(*bits)))), + F64 => packer.push(Datum::from(f64::from_bits(*bits))), + STR | JSON => packer.push(Datum::String(if bits % 2 == 0 { "a" } else { "b" })), + BOOL => packer.push(Datum::from(*bits % 2 == 0)), + _ => unreachable!(), + }, + FuzzDatum::Bytes(bytes) => match col { + STR => packer.push(Datum::String( + std::str::from_utf8(bytes).unwrap_or("\u{fffd}"), + )), + _ => push_pool(packer, col, bytes[0]), + }, + FuzzDatum::Pool(idx) => push_pool(packer, col, *idx), + } +} + +fn push_pool(packer: &mut mz_repr::RowPacker, col: usize, idx: u8) { + let pool: Vec> = col_type(col).interesting_datums().collect(); + if pool.is_empty() { + packer.push(Datum::Null); + } else { + packer.push(pool[idx as usize % pool.len()]); + } +} + +fn lit(col: usize, d: &FuzzDatum) -> MirScalarExpr { + let mut row = Row::default(); + push_datum(&mut row.packer(), col, d); + let datum = row.iter().next().unwrap(); + if datum.is_null() { + MirScalarExpr::literal_null(repr_type(col)) + } else { + MirScalarExpr::literal_ok(datum, repr_type(col)) + } +} + +fn cmp_func(cmp: &Cmp) -> BinaryFunc { + match cmp { + Cmp::Lt => BinaryFunc::Lt(Lt), + Cmp::Lte => BinaryFunc::Lte(Lte), + Cmp::Gt => BinaryFunc::Gt(Gt), + Cmp::Gte => BinaryFunc::Gte(Gte), + Cmp::Eq => BinaryFunc::Eq(Eq), + } +} + +fn binary(func: BinaryFunc, a: MirScalarExpr, b: MirScalarExpr) -> MirScalarExpr { + MirScalarExpr::CallBinary { + func, + expr1: Box::new(a), + expr2: Box::new(b), + } +} + +fn f64_lit(bits: u64) -> MirScalarExpr { + MirScalarExpr::literal_ok(Datum::from(f64::from_bits(bits)), ReprScalarType::Float64) +} + +fn build_pred(pred: &Pred) -> MirScalarExpr { + match pred { + Pred::CmpColLit { col, cmp, lit: l } => { + let col = *col as usize % ARITY; + binary(cmp_func(cmp), MirScalarExpr::column(col), lit(col, l)) + } + Pred::IsNull { col, negate } => { + let expr = MirScalarExpr::CallUnary { + func: UnaryFunc::IsNull(IsNull), + expr: Box::new(MirScalarExpr::column(*col as usize % ARITY)), + }; + if *negate { + MirScalarExpr::CallUnary { + func: UnaryFunc::Not(Not), + expr: Box::new(expr), + } + } else { + expr + } + } + Pred::JsonbKey { key, stringify } => { + let keys = ["x", "y", "nested", "absent"]; + let func = if *stringify { + BinaryFunc::JsonbGetStringStringify(JsonbGetStringStringify) + } else { + BinaryFunc::JsonbGetString(JsonbGetString) + }; + let get = binary( + func, + MirScalarExpr::column(JSON), + MirScalarExpr::literal_ok( + Datum::String(keys[*key as usize % keys.len()]), + ReprScalarType::String, + ), + ); + MirScalarExpr::CallUnary { + func: UnaryFunc::IsNull(IsNull), + expr: Box::new(get), + } + } + Pred::FloatArith { cmp, a, b, c } => { + let add = binary( + BinaryFunc::AddFloat64(AddFloat64), + MirScalarExpr::column(F64), + f64_lit(*a), + ); + let mul = binary(BinaryFunc::MulFloat64(MulFloat64), add, f64_lit(*b)); + binary(cmp_func(cmp), mul, f64_lit(*c)) + } + Pred::And(a, b) => MirScalarExpr::CallVariadic { + func: And.into(), + exprs: vec![build_pred(a), build_pred(b)], + }, + Pred::Or(a, b) => MirScalarExpr::CallVariadic { + func: Or.into(), + exprs: vec![build_pred(a), build_pred(b)], + }, + Pred::Not(a) => MirScalarExpr::CallUnary { + func: UnaryFunc::Not(Not), + expr: Box::new(build_pred(a)), + }, + } +} + +fn check(input: Input) { + let desc = schema(); + + let mut rows = Vec::new(); + for fuzz_row in input.rows.iter().take(8) { + let mut row = Row::default(); + let mut packer = row.packer(); + for (col, datum) in [ + &fuzz_row.num, + &fuzz_row.f64_bits, + &fuzz_row.string, + &fuzz_row.json, + &fuzz_row.bool_null, + ] + .into_iter() + .enumerate() + { + push_datum(&mut packer, col, datum); + } + drop(packer); + rows.push(SourceData(Ok(row))); + } + for _ in 0..input.err_rows.count_ones().min(2) { + rows.push(SourceData(Err(DataflowError::from( + mz_expr::EvalError::DivisionByZero, + )))); + } + if rows.is_empty() { + return; + } + + let mfp = MapFilterProject::new(ARITY).filter(std::iter::once(build_pred(&input.pred))); + let Ok(plan) = mfp.clone().into_plan() else { + return; + }; + + let mut builder = PartBuilder::new(&desc, &UnitSchema); + for row in &rows { + builder.push(row, &(), 1u64, 1i64); + } + let part = builder.finish(); + let part_stats = PartStats::new::(&part, &desc).expect("stats"); + let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); + let stats = RelationPartStats::new("fuzz", &metrics, &desc, &part_stats); + + // Ground truth: does any row produce output? Error rows always surface. + let arena = RowArena::new(); + let mut row_builder = Row::default(); + let yields_output = rows.iter().any(|source_data| match &source_data.0 { + Err(_) => true, + Ok(row) => { + let mut datums: Vec = row.iter().collect(); + plan.evaluate::( + &mut datums, + &arena, + Timestamp::MIN, + Diff::from(1), + |_| true, + &mut row_builder, + ) + .next() + .is_some() + } + }); + + if yields_output { + assert!( + stats.may_match_mfp(ResultSpec::anything(), &mfp), + "pushdown claims no row can match, but the MFP yields output on a real row \ + (wrongly-skipped part)\nrows={rows:?}\nmfp={mfp:?}", + ); + } +} + +fuzz_target!(|input: Input| check(input)); diff --git a/src/storage-types/src/stats.rs b/src/storage-types/src/stats.rs index 8858b99ad46f9..2b0d165bec209 100644 --- a/src/storage-types/src/stats.rs +++ b/src/storage-types/src/stats.rs @@ -229,7 +229,8 @@ mod tests { use mz_persist_types::codec_impls::UnitSchema; use mz_persist_types::columnar::{ColumnDecoder, Schema}; use mz_persist_types::part::PartBuilder; - use mz_persist_types::stats::PartStats; + use mz_persist_types::stats::{PartStats, ProtoStructStats, TrimStats, trim_to_budget}; + use mz_proto::RustType; use mz_repr::{Datum, RelationDesc, Row, RowArena, SqlColumnType, SqlScalarType}; use mz_repr::{SqlRelationType, arb_datum_for_column}; use proptest::prelude::*; @@ -256,19 +257,53 @@ mod tests { .expect("success"); let key_stats = decoder.stats(); - let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); - let stats = RelationPartStats { - name: "test", - metrics: &metrics, - stats: &PartStats { key: key_stats }, - desc: &schema, - }; - let arena = RowArena::default(); + // Trimming may widen bounds or drop them entirely, but must never + // narrow them, so the containment check below has to hold after every + // trimming pass: the lossy-but-column-preserving `trim`, and + // `trim_to_budget` at budgets all the way down to one that drops + // every column. The force-keep column matches the production default + // of never trimming the err column's stats. + let proto: ProtoStructStats = RustType::into_proto(&key_stats); + let mut variants = vec![("collected".to_string(), key_stats)]; + { + let mut trimmed = proto.clone(); + trimmed.trim(); + variants.push(( + "trimmed".to_string(), + RustType::from_proto(trimmed).expect("valid proto"), + )); + } + let full = prost::Message::encoded_len(&proto); + for budget in [full / 2, full / 4, 16, 0] { + let mut trimmed = proto.clone(); + trim_to_budget(&mut trimmed, budget, |col| col == "err"); + variants.push(( + format!("trim_to_budget({budget})"), + RustType::from_proto(trimmed).expect("valid proto"), + )); + } - // Validate that the stats would include all of the provided datums. - for datum in datums { - let spec = stats.col_stats(&ColumnIndex::from_raw(0), &arena); - assert!(spec.may_contain(*datum)); + let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); + for (label, key_stats) in variants { + let stats = RelationPartStats { + name: "test", + metrics: &metrics, + stats: &PartStats { key: key_stats }, + desc: &schema, + }; + let arena = RowArena::default(); + + // Validate that the stats would include all of the provided datums. + for datum in datums { + let spec = stats.col_stats(&ColumnIndex::from_raw(0), &arena); + if !spec.may_contain(*datum) { + return Err(format!( + "{label} stats-derived spec claims {datum:?} is absent from a part that \ + contains it (type: {:?}, part: {datums:?}, spec: {spec:?})", + column_type.scalar_type, + )); + } + } } Ok(()) @@ -298,6 +333,42 @@ mod tests { }); } + /// Deterministic sweep over multi-datum parts: every pair of interesting + /// datums and the full set, packed into a single part per type. + /// + /// Part bounds are computed over the whole part, so a value whose ordering + /// the stats collection disagrees on (e.g. -NaN, which arrow's total order + /// puts below -Infinity but `OrderedFloat` ranks above every finite value) + /// can invalidate the bounds for *other* values in the part. Single-datum + /// parts, as covered by `all_scalar_types_stats_roundtrip`, can never + /// catch that class of bug. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // too slow + fn interesting_datum_combinations_stats_roundtrip() { + for scalar_type in SqlScalarType::enumerate() { + let datums: Vec<_> = scalar_type.interesting_datums().collect(); + if datums.is_empty() { + continue; + } + for nullable in [false, true] { + let column_type = scalar_type.clone().nullable(nullable); + for (i, a) in datums.iter().enumerate() { + for b in &datums[i + 1..] { + assert_eq!(validate_stats(&column_type, &[*a, *b]), Ok(())); + } + if nullable { + assert_eq!(validate_stats(&column_type, &[*a, Datum::Null]), Ok(())); + } + } + let mut all = datums.clone(); + if nullable { + all.push(Datum::Null); + } + assert_eq!(validate_stats(&column_type, &all[..]), Ok(())); + } + } + } + #[mz_ore::test] #[cfg_attr(miri, ignore)] // too slow fn all_datums_produce_valid_stats() { diff --git a/test/cargo-fuzz/mzcompose.py b/test/cargo-fuzz/mzcompose.py index 0fc8814e5aead..ca9f049255119 100644 --- a/test/cargo-fuzz/mzcompose.py +++ b/test/cargo-fuzz/mzcompose.py @@ -432,6 +432,19 @@ def _reap(self, job: Job) -> None: if job.returncode == 0 and not self._new_artifacts(job): self.succeeded.append(job) say(f"✓ {job.name} [{secs}s] {final_stats(job.log_path)}") + elif ( + job.returncode is not None + and job.returncode < 0 + and not self._new_artifacts(job) + ): + # Killed by a signal (Ctrl-C, step timeout, an external kill) + # without a crash artifact: an interrupted run, not a crash. + # Crashes always leave an artifact, so this cannot mask one. + self.succeeded.append(job) + say( + f"- {job.name} interrupted by signal {-job.returncode} [{secs}s] " + f"{final_stats(job.log_path)}" + ) else: self.failed.append(job) say(self._failure_block(job, secs)) @@ -505,13 +518,15 @@ def _terminate_all(self, sig: int) -> None: except ProcessLookupError: pass - def build(self) -> None: - # Compile every fuzz crate up front, one at a time, not just the crates - # this run will fuzz. A fuzz target that won't compile is a broken - # build: building only the crates the active --profile/filters select - # would let a compile break in a skipped crate pass as a green run, - # since that crate is never compiled. Building all of them makes a - # broken fuzzer fail the run immediately, whatever the profile. + def build(self, crates: list[str] | None = None) -> None: + # By default compile every fuzz crate up front, one at a time, not + # just the crates this run will fuzz. A fuzz target that won't compile + # is a broken build: building only the crates the active --profile + # selects would let a compile break in a skipped crate pass as a green + # run, since that crate is never compiled. Building all of them makes + # a broken fuzzer fail the run immediately, whatever the profile. + # Explicit positional `filters` are the exception: those are targeted + # runs, and `crates` narrows the build to the crates they selected. # Sequential, one crate at a time, so the concurrent fuzzing phase # doesn't have 20+ `cargo fuzz run` invocations fighting over cargo's # per-target-dir build lock; crates share the target dir, so common @@ -525,8 +540,9 @@ def build(self) -> None: cmd = ["cargo", "fuzz", "build"] if self.sanitizer: cmd.append(f"--sanitizer={self.sanitizer}") - for i, crate in enumerate(FUZZ_CRATES, 1): - say(f"building [{i}/{len(FUZZ_CRATES)}] {crate}") + build_crates = FUZZ_CRATES if crates is None else crates + for i, crate in enumerate(build_crates, 1): + say(f"building [{i}/{len(build_crates)}] {crate}") if subprocess.run(cmd, cwd=MZ_ROOT / crate, env=self.env).returncode != 0: raise ui.UIError(f"build FAILED for {crate}") @@ -1114,7 +1130,10 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: for crate in shard_crates: prepare_corpus(crate, env) if not args.no_build: - runner.build() + # Explicit filters mean a targeted run: build only the crates whose + # targets were selected. Without filters, build everything so a + # compile break in any fuzz crate fails the run (see build()). + runner.build(crates=shard_crates if args.filters else None) failed = runner.run() upload_logs(env, log_dir) if args.corpus_sync: diff --git a/test/pgtest-mz/datums.pt b/test/pgtest-mz/datums.pt index bec32dbac9d56..36510620c0f8b 100644 --- a/test/pgtest-mz/datums.pt +++ b/test/pgtest-mz/datums.pt @@ -25,31 +25,35 @@ ReadyForQuery {"status":"I"} RowDescription {"fields":[{"name":"rowid"},{"name":"_bool"},{"name":"_int16"},{"name":"_int32"},{"name":"_int64"},{"name":"_uint16"},{"name":"_uint32"},{"name":"_uint64"},{"name":"_float32"},{"name":"_float64"},{"name":"_numeric"},{"name":"_date"},{"name":"_time"},{"name":"_timestamp"},{"name":"_timestamp_"},{"name":"_timestamp__"},{"name":"_timestamptz"},{"name":"_timestamptz_"},{"name":"_timestamptz__"},{"name":"_interval"},{"name":"_pglegacychar"},{"name":"_bytes"},{"name":"_string"},{"name":"_char"},{"name":"_varchar"},{"name":"_jsonb"},{"name":"_uuid"},{"name":"_oid"},{"name":"_regproc"},{"name":"_regtype"},{"name":"_regclass"},{"name":"_int2vector"},{"name":"_mztimestamp"},{"name":"_mzaclitem"}]} DataRow {"fields":["1","t","0","0","0","0","0","0","0","0","0","2000-01-01","00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00+00","1970-01-01 00:00:00+00","1970-01-01 00:00:00+00","00:00:00","\u0000","\\x",""," ","","true","00000000-0000-0000-0000-000000000000","0","0","0","0","NULL","0","=/p"]} DataRow {"fields":["2","f","1","1","1","1","1","1","1","1","1","4714-11-24 BC","23:59:59.999999","4714-12-31 00:00:00 BC","4714-12-31 00:00:00 BC","4714-12-31 00:00:00 BC","4714-12-31 00:00:00+00 BC","4714-12-31 00:00:00+00 BC","4714-12-31 00:00:00+00 BC","1 mon 1 day 00:00:00.000001","[255]","\\x00"," ","'"," ","false","ffffffff-ffff-ffff-ffff-ffffffffffff","4294967295","4294967295","4294967295","4294967295","NULL","18446744073709551615","=arwdUCRBNP/p"]} -DataRow {"fields":["3","NULL","-1","-1","-1","65535","4294967295","18446744073709551615","-1","-1","-1","262142-12-31","NULL","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","-1 mons -1 days -00:00:00.000001","NULL","\\xff","'","\"","'","null","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u42=/p"]} +DataRow {"fields":["3","NULL","-1","-1","-1","65535","4294967295","18446744073709551615","-1","-1","-1","262142-12-31","23:59:60.1999999","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","-1 mons -1 days -00:00:00.000001","NULL","\\xff","'","\"","'","null","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u42=/p"]} DataRow {"fields":["4","NULL","-32768","-2147483648","-9223372036854775808","255","32767","2147483647","-3.4028235e+38","-1.7976931348623157e+308","-Infinity","NULL","NULL","1970-01-01 00:00:00.123457","1970-01-01 00:00:00.123457","1970-01-01 00:00:00.123457","1970-01-01 00:00:00.123457+00","1970-01-01 00:00:00.123457+00","1970-01-01 00:00:00.123457+00","1 mon","NULL","NULL","\"",".","\"","\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u42=arwdUCRBNP/p"]} DataRow {"fields":["5","NULL","-32767","-2147483647","-9223372036854775807","256","32768","2147483648","1.1754944e-38","2.2250738585072014e-308","0","NULL","NULL","2019-07-24 23:59:60.234","2019-07-24 23:59:60.234","2019-07-24 23:59:60.234","NULL","NULL","NULL","1 day","NULL","NULL",".",",",".","\" \"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","=/u42"]} DataRow {"fields":["6","NULL","32767","2147483647","9223372036854775807","NULL","NULL","NULL","3.4028235e+38","1.7976931348623157e+308","Infinity","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","00:00:00.000001","NULL","NULL","2015-09-18T23:56:04.123Z","\t","2015-09-18T23:56:04.123Z","\"'\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","=arwdUCRBNP/u42"]} DataRow {"fields":["7","NULL","127","32767","2147483647","NULL","NULL","NULL","1.1920929e-7","2.220446049250313e-16","0.0000000000000002220446049250313","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-1 mons","NULL","NULL","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","\n","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","\"\\\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["8","NULL","128","32768","2147483648","NULL","NULL","NULL","NaN","NaN","NaN","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-1 days","NULL","NULL","JAPAN","\r","JAPAN","\".\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["9","NULL","NULL","NULL","NULL","NULL","NULL","NULL","Infinity","Infinity","Infinity","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-00:00:00.000001","NULL","NULL","1,2,3","\\","1,2,3","\"2015-09-18T23:56:04.123Z\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["10","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-Infinity","-Infinity","-Infinity","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-178956970 years -8 mons -2147483648 days -2562047788:00:54.775808","NULL","NULL","\r\n","\u0000","\r\n","\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["11","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","178956970 years 7 mons 2147483647 days 2562047788:00:54.775807","NULL","NULL","\"\"","\u0002","\"\"","\"JAPAN\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["12","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-178956970 years -8 mons","NULL","NULL"," ","\u0003"," ","\"1,2,3\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["13","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","178956970 years 7 mons","NULL","NULL","'","\b","'","\"\\r\\n\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["14","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-2147483648 days","NULL","NULL","\"","\u001b","\"","\"\\\"\\\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["15","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","2147483647 days","NULL","NULL",".","",".","0","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["16","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-2562047788:00:54.775808","NULL","NULL",",","NULL",",","1","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["17","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","2562047788:00:54.775807","NULL","NULL","\t","NULL","\t","-1","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["18","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\n","NULL","\n","0","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["19","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\r","NULL","\r","0.0000000000000002220446049250313","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["20","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\\","NULL","\\","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["21","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0000","NULL","\u0000","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["22","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0002","NULL","\u0002","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["23","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0003","NULL","\u0003","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["24","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\b","NULL","\b","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["25","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u001b","NULL","\u001b","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["26","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","","NULL","","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -CommandComplete {"tag":"SELECT 26"} +DataRow {"fields":["8","NULL","128","32768","2147483648","NULL","NULL","NULL","NaN","NaN","NaN","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-1 days","NULL","NULL","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","\r","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","\".\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["9","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NaN","NaN","Infinity","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-00:00:00.000001","NULL","NULL","􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿","\\","􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿","\"2015-09-18T23:56:04.123Z\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["10","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-0","-0","-Infinity","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-178956970 years -8 mons -2147483648 days -2562047788:00:54.775808","NULL","NULL","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx😀","\u0000","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx😀","\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["11","NULL","NULL","NULL","NULL","NULL","NULL","NULL","Infinity","Infinity","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","178956970 years 7 mons 2147483647 days 2562047788:00:54.775807","NULL","NULL","JAPAN","\u0002","JAPAN","\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["12","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-Infinity","-Infinity","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-178956970 years -8 mons","NULL","NULL","1,2,3","\u0003","1,2,3","\"􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["13","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","178956970 years 7 mons","NULL","NULL","\r\n","\b","\r\n","\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx😀\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["14","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-2147483648 days","NULL","NULL","\"\"","\u001b","\"\"","\"JAPAN\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["15","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","2147483647 days","NULL","NULL"," ",""," ","\"1,2,3\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["16","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","-2562047788:00:54.775808","NULL","NULL","'","NULL","'","\"\\r\\n\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["17","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","2562047788:00:54.775807","NULL","NULL","\"","NULL","\"","\"\\\"\\\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["18","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL",".","NULL",".","0","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["19","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL",",","NULL",",","1","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["20","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\t","NULL","\t","-1","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["21","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\n","NULL","\n","0","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["22","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\r","NULL","\r","0.0000000000000002220446049250313","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["23","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\\","NULL","\\","{\"x\":\"a\"}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["24","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0000","NULL","\u0000","{\"y\":\"b\"}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["25","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0002","NULL","\u0002","{\"x\":true,\"y\":null}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["26","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0003","NULL","\u0003","{}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["27","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\b","NULL","\b","{\"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk\":true}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["28","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u001b","NULL","\u001b","{\"nested\":{\"x\":\"a\"}}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["29","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","","NULL","","[true,null,\"a\"]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["30","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +CommandComplete {"tag":"SELECT 30"} ReadyForQuery {"status":"I"} # Binary @@ -67,29 +71,33 @@ ParseComplete BindComplete DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","\u0001","\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000","",""," ","","\u0001true","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","NULL","0","p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]} DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0002","\u0000","\u0000\u0001","\u0000\u0000\u0000\u0001","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","\u0000\u0001","\u0000\u0000\u0000\u0001","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","[63, 128, 0, 0]","[63, 240, 0, 0, 0, 0, 0, 0]","\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","[255, 218, 151, 167]","[0, 0, 0, 20, 29, 215, 95, 255]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0001","[255]","\u0000"," ","'"," ","\u0001false","[255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255]","NULL","18446744073709551615","[112, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 224, 1, 0, 0, 0]"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003","NULL","[255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255, 255, 255, 255, 255]","[255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255, 255, 255, 255, 255]","[191, 128, 0, 0]","[191, 240, 0, 0, 0, 0, 0, 0]","\u0000\u0001\u0000\u0000@\u0000\u0000\u0000\u0000\u0001","[5, 169, 209, 111]","NULL","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255]","NULL","[255]","'","\"","'","\u0001null","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u*\u0000\u0000\u0000\u0000\u0000\u0000\u0000p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003","NULL","[255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255, 255, 255, 255, 255]","[255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255, 255, 255, 255, 255]","[191, 128, 0, 0]","[191, 240, 0, 0, 0, 0, 0, 0]","\u0000\u0001\u0000\u0000@\u0000\u0000\u0000\u0000\u0001","[5, 169, 209, 111]","[0, 0, 0, 20, 29, 230, 162, 63]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255]","NULL","[255]","'","\"","'","\u0001null","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u*\u0000\u0000\u0000\u0000\u0000\u0000\u0000p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]} DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0004","NULL","[128, 0]","[128, 0, 0, 0]","[128, 0, 0, 0, 0, 0, 0, 0]","[0, 255]","[0, 0, 127, 255]","[0, 0, 0, 0, 127, 255, 255, 255]","[255, 127, 255, 255]","[255, 239, 255, 255, 255, 255, 255, 255]","[0, 0, 255, 255, 240, 0, 0, 0]","NULL","NULL","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","NULL","NULL","\"",".","\"","\u0001\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[117, 42, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 224, 1, 0, 0, 0]"]} DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0005","NULL","[128, 1]","[128, 0, 0, 1]","[128, 0, 0, 0, 0, 0, 0, 1]","\u0001\u0000","[0, 0, 128, 0]","[0, 0, 0, 0, 128, 0, 0, 0]","[0, 128, 0, 0]","\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","NULL","NULL","[0, 2, 49, 116, 224, 41, 242, 16]","[0, 2, 49, 116, 224, 41, 242, 16]","[0, 2, 49, 116, 224, 41, 242, 16]","NULL","NULL","NULL","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000","NULL","NULL",".",",",".","\u0001\" \"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000u*\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]} DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0006","NULL","[127, 255]","[127, 255, 255, 255]","[127, 255, 255, 255, 255, 255, 255, 255]","NULL","NULL","NULL","[127, 127, 255, 255]","[127, 239, 255, 255, 255, 255, 255, 255]","[0, 0, 255, 255, 208, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","NULL","NULL","2015-09-18T23:56:04.123Z","\t","2015-09-18T23:56:04.123Z","\u0001\"'\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[112, 0, 0, 0, 0, 0, 0, 0, 0, 117, 42, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 224, 1, 0, 0, 0]"]} DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0007","NULL","\u0000","[0, 0, 127, 255]","[0, 0, 0, 0, 127, 255, 255, 255]","NULL","NULL","NULL","4\u0000\u0000\u0000","[60, 176, 0, 0, 0, 0, 0, 0]","[0, 9, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 156, 17, 252, 36, 34, 12, 58]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255]","NULL","NULL","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","\n","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","\u0001\"\\\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\b","NULL","[0, 128]","[0, 0, 128, 0]","[0, 0, 0, 0, 128, 0, 0, 0]","NULL","NULL","NULL","[127, 192, 0, 0]","[127, 248, 0, 0, 0, 0, 0, 0]","[0, 0, 255, 255, 192, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0]","NULL","NULL","JAPAN","\r","JAPAN","\u0001\".\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\t","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[127, 128, 0, 0]","[127, 240, 0, 0, 0, 0, 0, 0]","[0, 0, 255, 255, 208, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0]","NULL","NULL","1,2,3","\\","1,2,3","\u0001\"2015-09-18T23:56:04.123Z\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\n","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[255, 128, 0, 0]","[255, 240, 0, 0, 0, 0, 0, 0]","[0, 0, 255, 255, 240, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0]","NULL","NULL","\r\n","\u0000","\r\n","\u0001\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u000b","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[127, 255, 255, 255, 255, 255, 255, 255, 127, 255, 255, 255, 127, 255, 255, 255]","NULL","NULL","\"\"","\u0002","\"\"","\u0001\"JAPAN\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\f","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0]","NULL","NULL"," ","\u0003"," ","\u0001\"1,2,3\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\r","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 255, 255, 255]","NULL","NULL","'","\b","'","\u0001\"\\r\\n\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u000e","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0]","NULL","NULL","\"","\u001b","\"","\u0001\"\\\"\\\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u000f","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 127, 255, 255, 255, 0, 0, 0, 0]","NULL","NULL",".","",".","\u00010","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0010","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","NULL","NULL",",","NULL",",","\u00011","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0011","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[127, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0]","NULL","NULL","\t","NULL","\t","\u0001-1","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0012","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\n","NULL","\n","\u00010","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0013","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\r","NULL","\r","\u00010.0000000000000002220446049250313","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\\","NULL","\\","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0015","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0000","NULL","\u0000","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0016","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0002","NULL","\u0002","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0017","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0003","NULL","\u0003","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0018","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\b","NULL","\b","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0019","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u001b","NULL","\u001b","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001a","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","","NULL","","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} -CommandComplete {"tag":"SELECT 26"} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\b","NULL","[0, 128]","[0, 0, 128, 0]","[0, 0, 0, 0, 128, 0, 0, 0]","NULL","NULL","NULL","[127, 192, 0, 0]","[127, 248, 0, 0, 0, 0, 0, 0]","[0, 0, 255, 255, 192, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0]","NULL","NULL","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","\r","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","\u0001\".\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\t","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[255, 192, 0, 0]","[255, 248, 0, 0, 0, 0, 0, 0]","[0, 0, 255, 255, 208, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0]","NULL","NULL","􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿","\\","􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿","\u0001\"2015-09-18T23:56:04.123Z\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\n","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[128, 0, 0, 0]","[128, 0, 0, 0, 0, 0, 0, 0]","[0, 0, 255, 255, 240, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0]","NULL","NULL","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx😀","\u0000","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx😀","\u0001\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u000b","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[127, 128, 0, 0]","[127, 240, 0, 0, 0, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[127, 255, 255, 255, 255, 255, 255, 255, 127, 255, 255, 255, 127, 255, 255, 255]","NULL","NULL","JAPAN","\u0002","JAPAN","\u0001\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\f","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[255, 128, 0, 0]","[255, 240, 0, 0, 0, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0]","NULL","NULL","1,2,3","\u0003","1,2,3","\u0001\"􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿􏿿\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\r","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 255, 255, 255]","NULL","NULL","\r\n","\b","\r\n","\u0001\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx😀\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u000e","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0]","NULL","NULL","\"\"","\u001b","\"\"","\u0001\"JAPAN\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u000f","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[0, 0, 0, 0, 0, 0, 0, 0, 127, 255, 255, 255, 0, 0, 0, 0]","NULL","NULL"," ",""," ","\u0001\"1,2,3\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0010","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","NULL","NULL","'","NULL","'","\u0001\"\\r\\n\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0011","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[127, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0]","NULL","NULL","\"","NULL","\"","\u0001\"\\\"\\\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0012","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL",".","NULL",".","\u00010","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0013","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL",",","NULL",",","\u00011","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\t","NULL","\t","\u0001-1","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0015","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\n","NULL","\n","\u00010","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0016","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\r","NULL","\r","\u00010.0000000000000002220446049250313","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0017","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\\","NULL","\\","\u0001{\"x\":\"a\"}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0018","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0000","NULL","\u0000","\u0001{\"y\":\"b\"}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0019","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0002","NULL","\u0002","\u0001{\"x\":true,\"y\":null}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001a","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0003","NULL","\u0003","\u0001{}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\b","NULL","\b","\u0001{\"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk\":true}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001c","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u001b","NULL","\u001b","\u0001{\"nested\":{\"x\":\"a\"}}","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001d","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","","NULL","","\u0001[true,null,\"a\"]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001e","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0001[]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL"]} +CommandComplete {"tag":"SELECT 30"} ReadyForQuery {"status":"I"} From a698442e5f4c93d30477c025318e271ad10fca9d Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 21:46:16 +0000 Subject: [PATCH 09/11] expr: document why range_lower's monotone claim survives its NULLs range_lower maps empty and unbounded-lower ranges to NULL, which the interpreter's endpoint box cannot represent. The claim is sound only because those inputs form a downward-closed prefix of the range ordering, so valued endpoints imply no NULL-yielding interior. Record that argument at the declaration and pin the SQL results, verified empirically (range columns also collect no statistics today, so pushdown cannot prune on them). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/expr/src/scalar/func/impls/range.rs | 9 +++++++++ test/sqllogictest/explain/pushdown.slt | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/expr/src/scalar/func/impls/range.rs b/src/expr/src/scalar/func/impls/range.rs index 8a5530df23a9e..6355d6d568d4e 100644 --- a/src/expr/src/scalar/func/impls/range.rs +++ b/src/expr/src/scalar/func/impls/range.rs @@ -84,6 +84,15 @@ impl fmt::Display for CastRangeToString { } } +// The monotone claim survives this function mapping empty and +// unbounded-lower ranges to NULL, which the interpreter's endpoint box +// cannot represent, only because those inputs form a downward-closed +// prefix of the range ordering (`None` inner sorts below `Some`, and a +// `None` lower bound sorts below every finite one): a range whose +// endpoints both yield values contains no NULL-yielding interior. Any +// change to range ordering or to this function's NULL cases must revisit +// the claim; see `try_parse_monotonic_iso8601_timestamp` for the +// SpecialUnary alternative. #[sqlfunc(sqlname = "rangelower", is_monotone = true)] fn range_lower(a: Range) -> Option { a.inner.map(|inner| inner.lower.bound).flatten() diff --git a/test/sqllogictest/explain/pushdown.slt b/test/sqllogictest/explain/pushdown.slt index 386bfcc305dca..87c4c59d495e6 100644 --- a/test/sqllogictest/explain/pushdown.slt +++ b/test/sqllogictest/explain/pushdown.slt @@ -214,6 +214,28 @@ SELECT DATE '2015-06-30' + TIME '23:59:60' = TIMESTAMP '2015-07-01 00:00:00' ---- true +# lower() maps empty and unbounded ranges to NULL; its declared +# monotonicity is sound because those sort below every value-yielding +# range. Range columns collect no statistics today, so pushdown cannot +# prune on them, but pin the results so a future stats addition revisits +# the NULL cases. + +statement ok +CREATE TABLE rt (r int4range) + +statement ok +INSERT INTO rt VALUES ('empty'), ('(,5)'), ('[1,5)'), ('[3,7)') + +query I +SELECT count(*) FROM rt WHERE lower(r) = 1 +---- +1 + +query I +SELECT count(*) FROM rt WHERE lower(r) IS NULL +---- +2 + # EXPLAIN FILTER PUSHDOWN FOR MATERIALIZED VIEW is also supported statement ok From 1b41a2f17cc05a89ce7b733109781ea452e9aa8d Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Fri, 7 Aug 2026 05:47:53 +0000 Subject: [PATCH 10/11] storage-types: don't underflow err_count on corrupt ok counts err_count subtracted the ok count from the part length unchecked: err stats whose none count exceeds the part length (corrupt or version-skewed durable state, the same class the previous hardening covers) underflowed and panicked the replica. Report the err count as unknown instead, which callers treat as may-error. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q5GiWioZM6CWrwWgfBADDR --- src/storage-types/src/stats.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/storage-types/src/stats.rs b/src/storage-types/src/stats.rs index 2b0d165bec209..dc97d214e0133 100644 --- a/src/storage-types/src/stats.rs +++ b/src/storage-types/src/stats.rs @@ -190,7 +190,9 @@ impl RelationPartStats<'_> { // then subtract that from the total. let num_results = self.stats.key.len; let num_oks = self.ok_count(); - num_oks.map(|num_oks| num_results - num_oks) + // An ok count exceeding the part length is corrupt stats; report the + // err count as unknown (callers keep the part) instead of underflowing. + num_oks.and_then(|num_oks| num_results.checked_sub(num_oks)) } fn col_values<'a>(&'a self, idx: &ColumnIndex, arena: &'a RowArena) -> Option> { @@ -490,6 +492,28 @@ mod tests { }; assert_eq!(stats.ok_count(), None); assert_eq!(stats.err_count(), None); + + // Well-shaped err stats whose none count exceeds the part length + // (corrupt or version-skewed) must read as unknown, not underflow. + let mut key_stats = decoder.stats(); + match key_stats.cols.get_mut("err") { + Some(err_stats) => match &mut err_stats.values { + ColumnStatKinds::Bytes(BytesStats::Primitive(_)) => { + err_stats.nulls = Some(mz_persist_types::stats::ColumnNullStats { + count: key_stats.len + 1, + }); + } + other => panic!("unexpected err stats {other:?}"), + }, + None => panic!("err stats missing"), + } + let stats = RelationPartStats { + name: "test", + metrics: &metrics, + stats: &PartStats { key: key_stats }, + desc: &schema, + }; + assert_eq!(stats.err_count(), None); } #[mz_ore::test] From 906adc79e17b1aab8f29036d4f4ba0ec9820f21e Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Fri, 7 Aug 2026 05:52:05 +0000 Subject: [PATCH 11/11] persist, storage, repr: address review feedback Finish the "fail open on undecodable part stats" change. `LazyPartStats` loses its panicking `decode`, so no read path can reintroduce the panic: the fast-path peek filter keeps the part, `EXPLAIN FILTER PUSHDOWN` reports it as selected, and inspect-state serializes the undecodable marker its `Debug` impl already renders. All three ran on stats straight off durable state, so version skew or corruption panicked clusterd and environmentd respectively. Extend that to the 'ok' column shape checks in `RelationPartStats`, which panicked on a wrong-shaped column on the very call stack the peek filter runs. A mismatch now counts in `mismatched_count` and reports the column range as unknown. Use the whole-second leap time as the interesting `TIME` datum. `from_hms_micro_opt(23, 59, 59, 1_999_999)` is a fractional leap second encoding to 86_400_999_999 microseconds, outside PostgreSQL's [0, 86_400_000_000] `time` domain, where a `postgres-types` client wraps it to 00:00:00.999999. Since `interesting_datums` also feeds the DATUMS load generator, that value reached a user-visible table. It is unreachable from SQL besides, which falsified the premise for keeping the leap representation in `TIME` at all. `TIME '23:59:60'` exercises the same `frac >= 1e9` path and encodes to exactly PostgreSQL's bound. Redact the `DataflowError` in the new error-row audit arm: those events go to Sentry, and `DecodeError` carries the raw source record bytes. Also drop the duplicated `value_between` doc summary line, document that the fuzz runner's interrupted classification treats a kernel OOM SIGKILL as interrupted too, relying on libFuzzer's rss limit to catch memory blowups as artifact-producing OOMs first, and replace the clippy-disallowed `prop_oneof!` and `Iterator::zip` in the new proptest strategies with `Union` and `zip_eq`. Co-Authored-By: Claude Fable 5 --- src/adapter/src/coord/sequencer.rs | 6 +- src/expr/src/interpret.rs | 1 - src/persist-client/src/internal/encoding.rs | 31 ++--- src/persist-client/src/internal/state.rs | 15 ++- src/pgcopy/src/copy.rs | 10 ++ src/repr/src/scalar.rs | 8 +- src/storage-operators/src/persist_source.rs | 65 +++++++---- src/storage-operators/src/stats.rs | 6 +- src/storage-types/src/stats.rs | 122 +++++++++++++++++++- test/cargo-fuzz/mzcompose.py | 5 +- test/pgtest-mz/datums.pt | 4 +- 11 files changed, 214 insertions(+), 59 deletions(-) diff --git a/src/adapter/src/coord/sequencer.rs b/src/adapter/src/coord/sequencer.rs index bb48eaf5149a2..4d057fc7e1121 100644 --- a/src/adapter/src/coord/sequencer.rs +++ b/src/adapter/src/coord/sequencer.rs @@ -1163,10 +1163,12 @@ pub(crate) async fn explain_pushdown_future_inner< let bytes = u64::cast_from(*bytes); total_bytes += bytes; total_parts += 1u64; - let selected = match stats { + let selected = match stats.as_ref().and_then(|x| x.try_decode().ok()) { + // Also the arm for stats that do not decode, which a + // newer writer's stats kind can produce. Both report the + // part as selected, matching what a read of it would do. None => true, Some(stats) => { - let stats = stats.decode(); let stats = RelationPartStats::new( name.as_str(), &snapshot_stats.metrics.pushdown.part_stats, diff --git a/src/expr/src/interpret.rs b/src/expr/src/interpret.rs index eb14a0868062b..0ea5390de5413 100644 --- a/src/expr/src/interpret.rs +++ b/src/expr/src/interpret.rs @@ -256,7 +256,6 @@ impl<'a> ResultSpec<'a> { } } - /// A spec that matches values between the given (non-null) min and max. /// A spec for the values between `min` and `max` inclusive. /// /// Unordered bounds widen to [`ResultSpec::value_all`] instead of collapsing diff --git a/src/persist-client/src/internal/encoding.rs b/src/persist-client/src/internal/encoding.rs index 8819c0f2bec73..ee3147cc2579c 100644 --- a/src/persist-client/src/internal/encoding.rs +++ b/src/persist-client/src/internal/encoding.rs @@ -1756,17 +1756,11 @@ impl LazyPartStats { /// This does not cache the returned value, it decodes each time it's /// called. /// - /// Panics if the encoded bytes are malformed. Only call this where the value - /// is known to have come from `Self::encode` rather than straight off blob. - pub fn decode(&self) -> PartStats { - self.try_decode().expect("valid stats") - } - - /// Like [Self::decode], but surfaces a malformed encoding as an error. - /// - /// The bytes are stored undecoded (see the [RustType] impl), so a corrupted - /// or crafted blob reaches here intact. Anything running on state that has - /// not been validated yet must use this. + /// The bytes are stored undecoded (see the [RustType] impl) and are never + /// validated on the way in, so a corrupted, crafted, or newer-version blob + /// reaches here intact. There is deliberately no infallible variant: every + /// caller reads stats straight off durable state, where a decode failure + /// must fail open (keep the part, report it selected) rather than panic. pub fn try_decode(&self) -> Result { let key = self .key @@ -2678,15 +2672,12 @@ mod tests { LazyPartStats::from_proto(Bytes::from(bytes)).expect("stats bytes are stored undecoded") } - /// `decode` panics on stats from a newer version, which is why the read - /// paths (the shard_source filter and the `stats()` accessors in fetch) - /// must use `try_decode` and fail open to fetching the part. - #[mz_ore::test] - #[should_panic(expected = "valid stats")] - fn part_stats_decode_panics_on_unknown_variant() { - let _ = version_skewed_part_stats().decode(); - } - + /// Stats from a newer version are an error rather than a value this + /// version misreads, which is what lets every read path fail open on + /// them: the `shard_source` filter and the fast-path peek filter keep the + /// part, the `stats()` accessors in fetch report `None`, `EXPLAIN FILTER + /// PUSHDOWN` reports the part as selected, and inspect-state serializes + /// the stats as absent. #[mz_ore::test] fn part_stats_try_decode_fails_open_on_unknown_variant() { assert_err!(version_skewed_part_stats().try_decode()); diff --git a/src/persist-client/src/internal/state.rs b/src/persist-client/src/internal/state.rs index 563d7a78af341..b63012c688e34 100644 --- a/src/persist-client/src/internal/state.rs +++ b/src/persist-client/src/internal/state.rs @@ -2740,8 +2740,19 @@ fn serialize_part_stats( val: &Option, s: S, ) -> Result { - let val = val.as_ref().map(|x| x.decode().key); - val.serialize(s) + // These bytes come from blob and are never validated on the way in, so a + // malformed or newer-version encoding reaches here intact. Report it as + // absent rather than panicking, and keep the field's shape stable for + // consumers of the inspect-state output by logging the failure instead of + // serializing a differently typed value in its place. + let stats = val.as_ref().and_then(|x| match x.try_decode() { + Ok(stats) => Some(stats.key), + Err(err) => { + tracing::warn!("undecodable part stats, reporting as absent: {err}"); + None + } + }); + stats.serialize(s) } fn serialize_diffs_sum(val: &Option<[u8; 8]>, s: S) -> Result { diff --git a/src/pgcopy/src/copy.rs b/src/pgcopy/src/copy.rs index 3228be736a5dd..a6fb01bd91095 100644 --- a/src/pgcopy/src/copy.rs +++ b/src/pgcopy/src/copy.rs @@ -1358,6 +1358,16 @@ mod tests { Datum::Timestamp(_) | Datum::TimestampTz(_) | Datum::Null => { continue; } + // Text carries no sign for NaN, so `-NaN` decodes as + // `NaN`, and `Row` equality compares the encoded bytes, + // which differ. The positive NaN of the interesting set + // covers the roundtrip itself. + Datum::Float32(f) if f.is_nan() && f.is_sign_negative() => { + continue; + } + Datum::Float64(f) if f.is_nan() && f.is_sign_negative() => { + continue; + } Datum::String(s) => { // TODO: The decoder cannot differentiate between empty string and null. if s.trim() == copy_csv_params.null || s.trim().is_empty() { diff --git a/src/repr/src/scalar.rs b/src/repr/src/scalar.rs index deb382e96e9ce..8d095a632cd06 100644 --- a/src/repr/src/scalar.rs +++ b/src/repr/src/scalar.rs @@ -4113,8 +4113,12 @@ impl SqlScalarType { Datum::Time(NaiveTime::from_hms_micro_opt(0, 0, 0, 0).unwrap()), Datum::Time(NaiveTime::from_hms_micro_opt(23, 59, 59, 999_999).unwrap()), // Leap second: chrono represents it as a fractional part of - // one second or more. - Datum::Time(NaiveTime::from_hms_micro_opt(23, 59, 59, 1_999_999).unwrap()), + // one second or more. `TIME '23:59:60'` is the largest value + // parsing admits, since fractional leap seconds are rejected, + // and it encodes to exactly PostgreSQL's 24:00:00 bound. A + // fractional leap second here would leave the type's + // PostgreSQL wire domain. + Datum::Time(NaiveTime::from_hms_micro_opt(23, 59, 59, 1_000_000).unwrap()), ]) }); static TIMESTAMP: LazyLock = LazyLock::new(|| { diff --git a/src/storage-operators/src/persist_source.rs b/src/storage-operators/src/persist_source.rs index f08e0cc28df49..ab10d44ce5e2b 100644 --- a/src/storage-operators/src/persist_source.rs +++ b/src/storage-operators/src/persist_source.rs @@ -711,10 +711,15 @@ impl PendingWork { sentry::with_scope( |scope| scope.set_tag("alert_id", "persist_pushdown_audit_violation"), || { + // `err` is redacted for the same reason the + // `Ok`-row arm redacts its MFP output: these + // events go to Sentry, and a `DecodeError` + // carries the raw source record bytes while + // several `EvalError`s embed user input. error!( ?stats, name, - ?err, + err = ?redact(&err), "persist filter pushdown correctness violation!" ); if self.panic_on_audit_failure { @@ -1534,6 +1539,7 @@ mod tests { /// column stat range that fails to contain a real value). See /// database-issues#9656 / PER-50. mod filter_pushdown_audit { + use itertools::Itertools; use mz_expr::func::variadic::{And, Or}; use mz_expr::func::{ AddFloat32, AddTimestampInterval, CastNumericToFloat32, CastNumericToMzTimestamp, Eq, @@ -1549,6 +1555,7 @@ mod tests { use mz_repr::{Diff, ReprScalarType, SqlScalarType}; use proptest::prelude::*; use proptest::sample::{Index, select}; + use proptest::strategy::Union; use super::*; @@ -1945,14 +1952,14 @@ mod tests { let ok_row = prop::collection::vec(any::(), WIDE_ARITY).prop_map(move |picks| { let datums = picks .iter() - .zip(&pools) + .zip_eq(&pools) .map(|(pick, pool)| pool[pick.index(pool.len())]); SourceData(Ok(Row::pack(datums))) }); let err_row = Just(SourceData(Err(DataflowError::from( EvalError::DivisionByZero, )))); - let row = prop_oneof![9 => ok_row, 1 => err_row]; + let row = Union::new_weighted(vec![(9, ok_row.boxed()), (1, err_row.boxed())]); prop::collection::vec(row, 2..8) } @@ -2154,15 +2161,15 @@ mod tests { } fn arb_wide_predicate() -> impl Strategy { - let leaf = prop_oneof![ - arb_cmp_col_lit(), - arb_is_null_pred(), - arb_jsonb_pred(), - arb_case_jsonb_pred(), - arb_iso_parse_pred(), - arb_ts_interval_pred(), - arb_float_mul_pred(), - arb_temporal_pred(), + let leaf = Union::new(vec![ + arb_cmp_col_lit().boxed(), + arb_is_null_pred().boxed(), + arb_jsonb_pred().boxed(), + arb_case_jsonb_pred().boxed(), + arb_iso_parse_pred().boxed(), + arb_ts_interval_pred().boxed(), + arb_float_mul_pred().boxed(), + arb_temporal_pred().boxed(), // The numeric shapes from the narrow test, aimed at the // fallible-interior mechanisms; both reference column 0. ( @@ -2170,21 +2177,31 @@ mod tests { f32_consts(), f32_consts(), f32_consts(), - comparison_funcs() + comparison_funcs(), ) - .prop_map(|(s, a, b, c, cmp)| float_arith_predicate(s, a, b, c, cmp)), + .prop_map(|(s, a, b, c, cmp)| float_arith_predicate(s, a, b, c, cmp)) + .boxed(), (select(vec![0u64, 1, 2, 100, u64::MAX]), comparison_funcs()) - .prop_map(|(ts, cmp)| cast_mz_timestamp_predicate(ts, cmp)), - ] + .prop_map(|(ts, cmp)| cast_mz_timestamp_predicate(ts, cmp)) + .boxed(), + ]) .boxed(); - prop_oneof![ - 3 => leaf.clone(), - 1 => (leaf.clone(), leaf.clone(), any::()).prop_map(|(a, b, is_and)| { - let func = if is_and { And.into() } else { Or.into() }; - MirScalarExpr::CallVariadic { func, exprs: vec![a, b] } - }), - 1 => leaf.prop_map(not), - ] + Union::new_weighted(vec![ + (3, leaf.clone()), + ( + 1, + (leaf.clone(), leaf.clone(), any::()) + .prop_map(|(a, b, is_and)| { + let func = if is_and { And.into() } else { Or.into() }; + MirScalarExpr::CallVariadic { + func, + exprs: vec![a, b], + } + }) + .boxed(), + ), + (1, leaf.prop_map(not).boxed()), + ]) } /// The zero-column count(*) path: when the read desc projects away diff --git a/src/storage-operators/src/stats.rs b/src/storage-operators/src/stats.rs index 6d45a75341963..1123ff2200f48 100644 --- a/src/storage-operators/src/stats.rs +++ b/src/storage-operators/src/stats.rs @@ -48,7 +48,11 @@ impl StatsCursor { let should_fetch = |name: &'static str, errors: bool| { move |stats: Option<&LazyPartStats>| { let Some(stats) = stats else { return true }; - let stats = stats.decode(); + // Stats written by a newer version may not decode. The sound + // fallback is to fetch the part. + let Ok(stats) = stats.try_decode() else { + return true; + }; let metrics = &metrics.pushdown.part_stats; let relation_stats = RelationPartStats::new(name, metrics, desc, &stats); if errors { diff --git a/src/storage-types/src/stats.rs b/src/storage-types/src/stats.rs index dc97d214e0133..06b6a0f351767 100644 --- a/src/storage-types/src/stats.rs +++ b/src/storage-types/src/stats.rs @@ -124,9 +124,20 @@ impl RelationPartStats<'_> { let typ = &self.desc.get_type(idx); let ok_stats = self.stats.key.col("ok")?; - let ok_stats = ok_stats - .try_as_optional_struct() - .expect("ok column should be nullable struct"); + // These stats come straight off durable state, so a corrupt or + // version-skewed encoding can carry any shape here. Report the column + // range as unknown rather than panicking the process reading it. + let ok_stats = match ok_stats.try_as_optional_struct() { + Ok(ok_stats) => ok_stats, + Err(err) => { + self.metrics.mismatched_count.inc(); + tracing::error!( + "expected nullable struct stats for the 'ok' column of {}: {err}", + self.name + ); + return None; + } + }; let col_stats = ok_stats.some.cols.get(name.as_str())?; if let SqlColumnType { @@ -200,8 +211,16 @@ impl RelationPartStats<'_> { let typ = self.desc.get_type(idx); let ok_stats = self.stats.key.cols.get("ok")?; + // See the note in `col_json`: durable stats can be any shape, so a + // wrong-shaped 'ok' column makes the range unknown, not a panic. let ColumnStatKinds::Struct(ok_stats) = &ok_stats.values else { - panic!("'ok' column stats should be a struct") + self.metrics.mismatched_count.inc(); + tracing::error!( + "expected struct stats for the 'ok' column of {}, found {:?}", + self.name, + ok_stats.values + ); + return None; }; let col_stats = ok_stats.cols.get(name.as_str())?; @@ -516,6 +535,101 @@ mod tests { assert_eq!(stats.err_count(), None); } + /// Wrong-shaped ok-column stats must read as "column range unknown", + /// which fails open to keeping the part. Both column paths run here: a + /// plain column goes through `col_values`, a JSON one adds `col_json`, + /// and neither may panic the process reading durable state. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // too slow + fn malformed_ok_stats_fail_open() { + use mz_expr::{BinaryFunc, MirScalarExpr, func}; + use mz_persist_types::stats::{ColumnNullStats, ColumnarStats, PrimitiveStats}; + use mz_repr::ReprScalarType; + + let schema = RelationDesc::builder() + .with_column("col", SqlScalarType::Int32.nullable(false)) + .with_column("json", SqlScalarType::Jsonb.nullable(true)) + .finish(); + let mut builder = PartBuilder::new(&schema, &UnitSchema); + builder.push( + &SourceData(Ok(Row::pack_slice(&[Datum::Int32(1), Datum::JsonNull]))), + &(), + 1u64, + 1i64, + ); + let part = builder.finish(); + let key_col = part.key.as_struct(); + let decoder = >::decoder(&schema, key_col.clone()) + .expect("success"); + + let json_idx = schema + .iter_all() + .map(|(idx, _name, _typ)| idx) + .nth(1) + .expect("two columns"); + // A filter no Ok row in this part satisfies. With well-shaped stats + // the part is skipped, so keeping it proves the fallback ran. + let mfp = MapFilterProject::new(2).filter(std::iter::once(MirScalarExpr::CallBinary { + func: BinaryFunc::Eq(func::Eq), + expr1: Box::new(MirScalarExpr::column(0)), + expr2: Box::new(MirScalarExpr::literal_ok( + Datum::Int32(999), + ReprScalarType::Int32, + )), + })); + + let metrics = PartStatsMetrics::new(&MetricsRegistry::new()); + let arena = RowArena::new(); + let well_shaped = PartStats { + key: decoder.stats(), + }; + let well_shaped = RelationPartStats::new("test", &metrics, &schema, &well_shaped); + assert!(!well_shaped.may_match_mfp(ResultSpec::anything(), &mfp)); + + // `col_values` matched the ok column's kind infallibly. A wrong kind + // makes every column's range unknown. + let mut not_a_struct = decoder.stats(); + not_a_struct.cols.insert( + "ok".to_string(), + ColumnarStats { + nulls: Some(ColumnNullStats { count: 0 }), + values: PrimitiveStats { + lower: 0i32, + upper: 0i32, + } + .into(), + }, + ); + let not_a_struct = PartStats { key: not_a_struct }; + let not_a_struct = RelationPartStats::new("test", &metrics, &schema, ¬_a_struct); + for (idx, _name, _typ) in schema.iter_all() { + assert_eq!(not_a_struct.col_stats(idx, &arena), ResultSpec::anything()); + } + assert!(not_a_struct.may_match_mfp(ResultSpec::anything(), &mfp)); + + // `col_json` additionally required the ok column to be nullable. A + // struct that is not widens the JSON range alone, so assert on that + // rather than on the part-level decision. + let mut not_nullable = decoder.stats(); + match not_nullable.cols.get_mut("ok") { + Some(ok_stats) => ok_stats.nulls = None, + None => panic!("ok stats missing"), + } + let not_nullable = PartStats { key: not_nullable }; + let not_nullable = RelationPartStats::new("test", &metrics, &schema, ¬_nullable); + let other_json = Datum::String("a"); + assert!( + !well_shaped + .col_stats(json_idx, &arena) + .may_contain(other_json) + ); + assert!( + not_nullable + .col_stats(json_idx, &arena) + .may_contain(other_json) + ); + } + #[mz_ore::test] #[ignore] // TODO(parkmycar): Re-enable this test with a smaller sample size. fn statistics_stability() { diff --git a/test/cargo-fuzz/mzcompose.py b/test/cargo-fuzz/mzcompose.py index ca9f049255119..c68735c34e794 100644 --- a/test/cargo-fuzz/mzcompose.py +++ b/test/cargo-fuzz/mzcompose.py @@ -439,7 +439,10 @@ def _reap(self, job: Job) -> None: ): # Killed by a signal (Ctrl-C, step timeout, an external kill) # without a crash artifact: an interrupted run, not a crash. - # Crashes always leave an artifact, so this cannot mask one. + # libFuzzer-detected crashes always leave an artifact, so this + # cannot mask one. A kernel OOM SIGKILL is also reported as + # interrupted; the rss limit passed to libFuzzer catches memory + # blowups as artifact-producing OOMs well before the kernel does. self.succeeded.append(job) say( f"- {job.name} interrupted by signal {-job.returncode} [{secs}s] " diff --git a/test/pgtest-mz/datums.pt b/test/pgtest-mz/datums.pt index 36510620c0f8b..7a15d4a60a60d 100644 --- a/test/pgtest-mz/datums.pt +++ b/test/pgtest-mz/datums.pt @@ -25,7 +25,7 @@ ReadyForQuery {"status":"I"} RowDescription {"fields":[{"name":"rowid"},{"name":"_bool"},{"name":"_int16"},{"name":"_int32"},{"name":"_int64"},{"name":"_uint16"},{"name":"_uint32"},{"name":"_uint64"},{"name":"_float32"},{"name":"_float64"},{"name":"_numeric"},{"name":"_date"},{"name":"_time"},{"name":"_timestamp"},{"name":"_timestamp_"},{"name":"_timestamp__"},{"name":"_timestamptz"},{"name":"_timestamptz_"},{"name":"_timestamptz__"},{"name":"_interval"},{"name":"_pglegacychar"},{"name":"_bytes"},{"name":"_string"},{"name":"_char"},{"name":"_varchar"},{"name":"_jsonb"},{"name":"_uuid"},{"name":"_oid"},{"name":"_regproc"},{"name":"_regtype"},{"name":"_regclass"},{"name":"_int2vector"},{"name":"_mztimestamp"},{"name":"_mzaclitem"}]} DataRow {"fields":["1","t","0","0","0","0","0","0","0","0","0","2000-01-01","00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00+00","1970-01-01 00:00:00+00","1970-01-01 00:00:00+00","00:00:00","\u0000","\\x",""," ","","true","00000000-0000-0000-0000-000000000000","0","0","0","0","NULL","0","=/p"]} DataRow {"fields":["2","f","1","1","1","1","1","1","1","1","1","4714-11-24 BC","23:59:59.999999","4714-12-31 00:00:00 BC","4714-12-31 00:00:00 BC","4714-12-31 00:00:00 BC","4714-12-31 00:00:00+00 BC","4714-12-31 00:00:00+00 BC","4714-12-31 00:00:00+00 BC","1 mon 1 day 00:00:00.000001","[255]","\\x00"," ","'"," ","false","ffffffff-ffff-ffff-ffff-ffffffffffff","4294967295","4294967295","4294967295","4294967295","NULL","18446744073709551615","=arwdUCRBNP/p"]} -DataRow {"fields":["3","NULL","-1","-1","-1","65535","4294967295","18446744073709551615","-1","-1","-1","262142-12-31","23:59:60.1999999","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","-1 mons -1 days -00:00:00.000001","NULL","\\xff","'","\"","'","null","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u42=/p"]} +DataRow {"fields":["3","NULL","-1","-1","-1","65535","4294967295","18446744073709551615","-1","-1","-1","262142-12-31","23:59:60","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","-1 mons -1 days -00:00:00.000001","NULL","\\xff","'","\"","'","null","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u42=/p"]} DataRow {"fields":["4","NULL","-32768","-2147483648","-9223372036854775808","255","32767","2147483647","-3.4028235e+38","-1.7976931348623157e+308","-Infinity","NULL","NULL","1970-01-01 00:00:00.123457","1970-01-01 00:00:00.123457","1970-01-01 00:00:00.123457","1970-01-01 00:00:00.123457+00","1970-01-01 00:00:00.123457+00","1970-01-01 00:00:00.123457+00","1 mon","NULL","NULL","\"",".","\"","\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u42=arwdUCRBNP/p"]} DataRow {"fields":["5","NULL","-32767","-2147483647","-9223372036854775807","256","32768","2147483648","1.1754944e-38","2.2250738585072014e-308","0","NULL","NULL","2019-07-24 23:59:60.234","2019-07-24 23:59:60.234","2019-07-24 23:59:60.234","NULL","NULL","NULL","1 day","NULL","NULL",".",",",".","\" \"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","=/u42"]} DataRow {"fields":["6","NULL","32767","2147483647","9223372036854775807","NULL","NULL","NULL","3.4028235e+38","1.7976931348623157e+308","Infinity","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","00:00:00.000001","NULL","NULL","2015-09-18T23:56:04.123Z","\t","2015-09-18T23:56:04.123Z","\"'\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","=arwdUCRBNP/u42"]} @@ -71,7 +71,7 @@ ParseComplete BindComplete DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","\u0001","\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","[255, 252, 162, 254, 196, 200, 32, 0]","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000","",""," ","","\u0001true","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000","NULL","0","p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]} DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0002","\u0000","\u0000\u0001","\u0000\u0000\u0000\u0001","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","\u0000\u0001","\u0000\u0000\u0000\u0001","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","[63, 128, 0, 0]","[63, 240, 0, 0, 0, 0, 0, 0]","\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","[255, 218, 151, 167]","[0, 0, 0, 20, 29, 215, 95, 255]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","[253, 15, 127, 169, 145, 64, 128, 0]","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0001","[255]","\u0000"," ","'"," ","\u0001false","[255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255]","NULL","18446744073709551615","[112, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 224, 1, 0, 0, 0]"]} -DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003","NULL","[255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255, 255, 255, 255, 255]","[255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255, 255, 255, 255, 255]","[191, 128, 0, 0]","[191, 240, 0, 0, 0, 0, 0, 0]","\u0000\u0001\u0000\u0000@\u0000\u0000\u0000\u0000\u0001","[5, 169, 209, 111]","[0, 0, 0, 20, 29, 230, 162, 63]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255]","NULL","[255]","'","\"","'","\u0001null","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u*\u0000\u0000\u0000\u0000\u0000\u0000\u0000p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]} +DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003","NULL","[255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255, 255, 255, 255, 255]","[255, 255]","[255, 255, 255, 255]","[255, 255, 255, 255, 255, 255, 255, 255]","[191, 128, 0, 0]","[191, 240, 0, 0, 0, 0, 0, 0]","\u0000\u0001\u0000\u0000@\u0000\u0000\u0000\u0000\u0001","[5, 169, 209, 111]","[0, 0, 0, 20, 29, 215, 96, 0]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[113, 237, 93, 56, 67, 138, 189, 192]","[255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255]","NULL","[255]","'","\"","'","\u0001null","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u*\u0000\u0000\u0000\u0000\u0000\u0000\u0000p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]} DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0004","NULL","[128, 0]","[128, 0, 0, 0]","[128, 0, 0, 0, 0, 0, 0, 0]","[0, 255]","[0, 0, 127, 255]","[0, 0, 0, 0, 127, 255, 255, 255]","[255, 127, 255, 255]","[255, 239, 255, 255, 255, 255, 255, 255]","[0, 0, 255, 255, 240, 0, 0, 0]","NULL","NULL","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","[255, 252, 162, 254, 196, 202, 2, 65]","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001","NULL","NULL","\"",".","\"","\u0001\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[117, 42, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 224, 1, 0, 0, 0]"]} DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0005","NULL","[128, 1]","[128, 0, 0, 1]","[128, 0, 0, 0, 0, 0, 0, 1]","\u0001\u0000","[0, 0, 128, 0]","[0, 0, 0, 0, 128, 0, 0, 0]","[0, 128, 0, 0]","\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","NULL","NULL","[0, 2, 49, 116, 224, 41, 242, 16]","[0, 2, 49, 116, 224, 41, 242, 16]","[0, 2, 49, 116, 224, 41, 242, 16]","NULL","NULL","NULL","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000","NULL","NULL",".",",",".","\u0001\" \"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","p\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000u*\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]} DataRow {"fields":["\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0006","NULL","[127, 255]","[127, 255, 255, 255]","[127, 255, 255, 255, 255, 255, 255, 255]","NULL","NULL","NULL","[127, 127, 255, 255]","[127, 239, 255, 255, 255, 255, 255, 255]","[0, 0, 255, 255, 208, 0, 0, 0]","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","NULL","NULL","2015-09-18T23:56:04.123Z","\t","2015-09-18T23:56:04.123Z","\u0001\"'\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","[112, 0, 0, 0, 0, 0, 0, 0, 0, 117, 42, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 224, 1, 0, 0, 0]"]}