Skip to content

Commit 9c1bd46

Browse files
frankmcsherryclaude
andcommitted
compute: key TopK input for its first stage in the initial map
The initial map hashed each row and packed (hash, group), and the first stage mapped that to (hash % modulus, group) and repacked. Compute the first stage's bucket in the initial map, zero when the only stage is the final one, and let a stage rekey only when its input is keyed for an earlier stage, as the min/max hierarchy already does. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 parent 08a0930 commit 9c1bd46

1 file changed

Lines changed: 27 additions & 14 deletions

File tree

src/compute/src/render/top_k.rs

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ impl<'scope, T: crate::render::RenderTimestamp + crate::render::MaybeBucketByTim
274274
let (result, errs) = self.build_topk_stage(
275275
thinned,
276276
order_key,
277-
1u64,
277+
Some(1),
278278
0,
279279
limit,
280280
arity,
@@ -354,13 +354,20 @@ impl<'scope, T: crate::render::RenderTimestamp + crate::render::MaybeBucketByTim
354354
) {
355355
let pairer = Pairer::new(1);
356356
let mut datum_vec = mz_repr::DatumVec::new();
357+
// Key each row for the first stage directly: its bucket of the row hash, or zero when
358+
// the only stage is the final one and the hash would be discarded anyway.
359+
let first_modulus = buckets.first().copied().unwrap_or(1);
357360
let mut collection = collection.map({
358361
move |row| {
359362
let group_row = {
360-
let row_hash = row.hashed();
363+
let bucket = if first_modulus == 1 {
364+
0
365+
} else {
366+
row.hashed() % first_modulus
367+
};
361368
let datums = datum_vec.borrow_with(&row);
362369
let iterator = group_key.iter().map(|i| datums[*i]);
363-
pairer.merge(std::iter::once(Datum::from(row_hash)), iterator)
370+
pairer.merge(std::iter::once(Datum::from(bucket)), iterator)
364371
};
365372
(group_row, row)
366373
}
@@ -388,6 +395,8 @@ impl<'scope, T: crate::render::RenderTimestamp + crate::render::MaybeBucketByTim
388395
StageOutput::Dropped
389396
}
390397
};
398+
// The input arrives keyed for the first stage; later stages rekey it.
399+
let mut first = true;
391400
let mut validating = true;
392401
let mut err_collection: Option<VecCollection<'s, T, _, _>> = None;
393402

@@ -426,13 +435,14 @@ impl<'scope, T: crate::render::RenderTimestamp + crate::render::MaybeBucketByTim
426435
let (oks, errs) = self.build_topk_stage(
427436
collection,
428437
order_key.clone(),
429-
bucket,
438+
(!first).then_some(bucket),
430439
0,
431440
Some(limit.clone()),
432441
arity,
433442
validating,
434443
stage_output(bucket),
435444
);
445+
first = false;
436446
collection = oks;
437447
if validating {
438448
err_collection = errs;
@@ -448,7 +458,7 @@ impl<'scope, T: crate::render::RenderTimestamp + crate::render::MaybeBucketByTim
448458
let (oks, errs) = self.build_topk_stage(
449459
collection,
450460
order_key,
451-
1u64,
461+
(!first).then_some(1),
452462
offset,
453463
limit,
454464
arity,
@@ -509,7 +519,7 @@ impl<'scope, T: crate::render::RenderTimestamp + crate::render::MaybeBucketByTim
509519
&self,
510520
collection: VecCollection<'s, T, (Row, Row), Diff>,
511521
order_key: Vec<mz_expr::ColumnOrder>,
512-
modulus: u64,
522+
modulus: Option<u64>,
513523
offset: usize,
514524
limit: Option<LirScalarExpr>,
515525
arity: usize,
@@ -519,14 +529,17 @@ impl<'scope, T: crate::render::RenderTimestamp + crate::render::MaybeBucketByTim
519529
VecCollection<'s, T, (Row, Row), Diff>,
520530
Option<VecCollection<'s, T, DataflowErrorSer, Diff>>,
521531
) {
522-
// Form appropriate input by updating the `hash` column (first datum in `hash_key`) by
523-
// applying `modulus`.
524-
let input = collection.map(move |(hash_key, row)| {
525-
let mut hash_key_iter = hash_key.iter();
526-
let hash = hash_key_iter.next().unwrap().unwrap_uint64() % modulus;
527-
let hash_key = SharedRow::pack(std::iter::once(hash.into()).chain(hash_key_iter));
528-
(hash_key, row)
529-
});
532+
// Rekey the input by applying `modulus` to the hash column (first datum in `hash_key`),
533+
// unless the input already arrives keyed for this stage.
534+
let input = match modulus {
535+
Some(modulus) => collection.map(move |(hash_key, row)| {
536+
let mut hash_key_iter = hash_key.iter();
537+
let hash = hash_key_iter.next().unwrap().unwrap_uint64() % modulus;
538+
let hash_key = SharedRow::pack(std::iter::once(hash.into()).chain(hash_key_iter));
539+
(hash_key, row)
540+
}),
541+
None => collection,
542+
};
530543

531544
// If validating: demux errors, otherwise we cannot produce errors.
532545
let (input, oks, errs) = if validating {

0 commit comments

Comments
 (0)