Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 13 additions & 21 deletions src/compute/src/render/columnar.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@ pub type ColumnarCollection<'scope, T, D, R> = Collection<'scope, T, Column<(D,
/// `concat`s repack the row-based inputs and produce the columnar variant.
#[derive(Clone)]
pub enum CollectionEdge<'scope, T: RenderTimestamp> {
/// Row-formatted collection. Today's default for every producer.
/// Row-formatted collection. No producer constructs this.
#[allow(dead_code)]
Vec(VecCollection<'scope, T, Row, Diff>),
/// Columnar collection. Currently unused by any producer; reserved for the
/// producer flip at the end of the migration.
Expand Down Expand Up @@ -123,28 +124,19 @@ impl<'scope, T: RenderTimestamp> CollectionEdge<'scope, T> {

/// Concatenates a collection of edges.
///
/// Edges of one shared variant concatenate natively. Mixed inputs upgrade
/// the row-based edges through [`vec_to_columnar`] and produce the
/// columnar variant. Repacking rows into columns copies bytes but
/// allocates no per-record `Row`s, so upgrading is the cheap direction.
/// Every input is columnar, so they concatenate natively.
pub fn concat_many<I>(scope: Scope<'scope, T>, edges: I) -> Self
where
I: IntoIterator<Item = Self>,
{
let mut vecs = Vec::new();
let mut cols = Vec::new();
for edge in edges {
match edge {
CollectionEdge::Vec(c) => vecs.push(c),
CollectionEdge::Columnar(c) => cols.push(c),
}
}
if cols.is_empty() {
CollectionEdge::Vec(differential_dataflow::collection::concatenate(scope, vecs))
} else {
cols.extend(vecs.into_iter().map(vec_to_columnar));
CollectionEdge::Columnar(differential_dataflow::collection::concatenate(scope, cols))
}
let cols = edges.into_iter().map(|edge| match edge {
CollectionEdge::Columnar(c) => c,
CollectionEdge::Vec(_) => unreachable!("no producer emits a `Vec` edge"),
});
CollectionEdge::Columnar(differential_dataflow::collection::concatenate(
scope,
cols.collect::<Vec<_>>(),
))
}

/// Applies `logic` to each record in this edge, exposing the record as a
Expand Down Expand Up @@ -505,7 +497,7 @@ mod tests {
}

#[mz_ore::test]
fn concat_many_mixed_upgrades_to_columnar() {
fn concat_many_concatenates_columnar() {
let rows = test_rows();
let expected: Vec<_> = {
let mut updates: Vec<_> = rows
Expand All @@ -524,7 +516,7 @@ mod tests {
let edge = CollectionEdge::concat_many(
scope,
[
CollectionEdge::Vec(collection1),
CollectionEdge::Columnar(vec_to_columnar(collection1)),
CollectionEdge::Columnar(vec_to_columnar(collection2)),
],
);
Expand Down
Loading