From 2188781f3a32dfaeac5942e2240217afe15cb8a0 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Mon, 13 Jan 2025 09:43:08 +0800 Subject: [PATCH 1/3] fix(query): increase state_rows in copy agg state rows --- src/query/expression/src/aggregate/payload.rs | 1 + .../pipelines/processors/transforms/window/transform_window.rs | 2 ++ src/query/sql/src/evaluator/block_operator.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/src/query/expression/src/aggregate/payload.rs b/src/query/expression/src/aggregate/payload.rs index 51cc3b579870e..562b2a899f9bf 100644 --- a/src/query/expression/src/aggregate/payload.rs +++ b/src/query/expression/src/aggregate/payload.rs @@ -350,6 +350,7 @@ impl Payload { ) } page.rows += 1; + page.state_rows += 1; if page.rows == page.capacity { (page, _) = self.writable_page(); diff --git a/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs b/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs index 6cb99f0159b4d..bdc2541586af0 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs @@ -449,6 +449,7 @@ impl TransformWindow { let mut output = block.block.clone(); let data_type = block.builder.data_type(); // The memory of the builder can be released. + println!("block.builder {}", block.builder.data_type()); let builder = std::mem::replace( &mut block.builder, ColumnBuilder::with_capacity(&data_type, 0), @@ -1327,6 +1328,7 @@ where T: Number + ResultTypeOfUnary } ProcessorState::Output => { let output = self.outputs.pop_front().unwrap(); + println!("output {}", output); self.output.push_data(Ok(output)); if self.outputs.is_empty() { self.state = ProcessorState::Consume; diff --git a/src/query/sql/src/evaluator/block_operator.rs b/src/query/sql/src/evaluator/block_operator.rs index 54740b86b634f..d6aefde77c2ee 100644 --- a/src/query/sql/src/evaluator/block_operator.rs +++ b/src/query/sql/src/evaluator/block_operator.rs @@ -157,6 +157,7 @@ impl Transform for CompoundBlockOperator { const SKIP_EMPTY_DATA_BLOCK: bool = true; fn transform(&mut self, data_block: DataBlock) -> Result { + println!("data_block -> {:?}", data_block); self.operators .iter() .try_fold(data_block, |input, op| op.execute(&self.ctx, input)) From 7e281ee8ea6fd58712831f1b1237980576e03936 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Mon, 13 Jan 2025 09:43:28 +0800 Subject: [PATCH 2/3] fix(query): increase state_rows in copy agg state rows --- .../pipelines/processors/transforms/window/transform_window.rs | 2 -- src/query/sql/src/evaluator/block_operator.rs | 1 - 2 files changed, 3 deletions(-) diff --git a/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs b/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs index bdc2541586af0..6cb99f0159b4d 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs @@ -449,7 +449,6 @@ impl TransformWindow { let mut output = block.block.clone(); let data_type = block.builder.data_type(); // The memory of the builder can be released. - println!("block.builder {}", block.builder.data_type()); let builder = std::mem::replace( &mut block.builder, ColumnBuilder::with_capacity(&data_type, 0), @@ -1328,7 +1327,6 @@ where T: Number + ResultTypeOfUnary } ProcessorState::Output => { let output = self.outputs.pop_front().unwrap(); - println!("output {}", output); self.output.push_data(Ok(output)); if self.outputs.is_empty() { self.state = ProcessorState::Consume; diff --git a/src/query/sql/src/evaluator/block_operator.rs b/src/query/sql/src/evaluator/block_operator.rs index d6aefde77c2ee..54740b86b634f 100644 --- a/src/query/sql/src/evaluator/block_operator.rs +++ b/src/query/sql/src/evaluator/block_operator.rs @@ -157,7 +157,6 @@ impl Transform for CompoundBlockOperator { const SKIP_EMPTY_DATA_BLOCK: bool = true; fn transform(&mut self, data_block: DataBlock) -> Result { - println!("data_block -> {:?}", data_block); self.operators .iter() .try_fold(data_block, |input, op| op.execute(&self.ctx, input)) From c9dc22bd3ccd662d64ba45beb15da5b844555d82 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Mon, 13 Jan 2025 10:16:38 +0800 Subject: [PATCH 3/3] fix(query): increase state_rows in copy agg state rows --- .../20+_others/20_0022_agg_memory.result | 6 +++ .../20+_others/20_0022_agg_memory.sh | 46 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100755 tests/suites/0_stateless/20+_others/20_0022_agg_memory.result create mode 100755 tests/suites/0_stateless/20+_others/20_0022_agg_memory.sh diff --git a/tests/suites/0_stateless/20+_others/20_0022_agg_memory.result b/tests/suites/0_stateless/20+_others/20_0022_agg_memory.result new file mode 100755 index 0000000000000..f32baf89a114b --- /dev/null +++ b/tests/suites/0_stateless/20+_others/20_0022_agg_memory.result @@ -0,0 +1,6 @@ +executing 1 +executing 2 +executing 3 +executing 4 +executing 5 +Memory usage difference is less than 5% diff --git a/tests/suites/0_stateless/20+_others/20_0022_agg_memory.sh b/tests/suites/0_stateless/20+_others/20_0022_agg_memory.sh new file mode 100755 index 0000000000000..e8602a4ae66a0 --- /dev/null +++ b/tests/suites/0_stateless/20+_others/20_0022_agg_memory.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../../../shell_env.sh + + +## warmup +for i in `seq 1 2`;do + $BENDSQL_CLIENT_CONNECT --query=""" + select number::string , max(number::string), min(number::string), count(distinct number) from numbers(10000000) group by 1 ignore_result; + """ +done + + +PIDS=($(pgrep databend-query)) +BEFORE_MEM=0 +for PID in "${PIDS[@]}"; do + MEM=$(ps -o rss= -p $PID | tail -n 1) + BEFORE_MEM=$((BEFORE_MEM + MEM)) +done + + +for i in `seq 1 5`;do + echo "executing $i" + $BENDSQL_CLIENT_CONNECT --query=""" + select number::string , max(number::string), min(number::string), count(distinct number) from numbers(10000000) group by 1 ignore_result; + """ +done + +sleep 15 + + +AFTER_MEM=0 +for PID in "${PIDS[@]}"; do + MEM=$(ps -o rss= -p $PID | tail -n 1) + AFTER_MEM=$((AFTER_MEM + MEM)) +done + +# Calculate the difference in percentage +DIFF=$(awk -v before=$BEFORE_MEM -v after=$AFTER_MEM 'BEGIN {print (after-before)/before * 100}') + +# Check if the difference is less than 5% +if (( $(awk -v diff=$DIFF 'BEGIN {print (diff < 5)}') )); then + echo "Memory usage difference is less than 5%" +else + echo "Memory usage difference is greater than 5%, before ${BEFORE_MEM} ${AFTER_MEM}" +fi \ No newline at end of file