Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
e541ef4
chore: add setting to provide fallback
dqhl76 Nov 7, 2024
c0ccbf9
refactor: make block pruning into pipeline
dqhl76 Nov 14, 2024
c2e9dd4
refactor: make block pruning into pipeline
dqhl76 Nov 19, 2024
713be70
fix: sender on finish unexpected close
dqhl76 Nov 19, 2024
c80b5b1
fix: take data block meta instead of clone
dqhl76 Nov 19, 2024
4f6d541
chore: remove dbg
dqhl76 Nov 19, 2024
e310557
chore: license header
dqhl76 Nov 19, 2024
3945c8d
fix: add back broken prune cache
dqhl76 Nov 20, 2024
2f6cd4e
Merge branch 'main' into split2
dqhl76 Nov 20, 2024
fd151b5
fix: populating cache only when pipeline successfully finished
dqhl76 Nov 21, 2024
4482cec
fix: broken statistics
dqhl76 Nov 22, 2024
62ff0da
fix: broken cache
dqhl76 Nov 25, 2024
0d16e20
Merge branch 'main' into split2
dqhl76 Nov 25, 2024
836d6f6
ensure can fallback
dqhl76 Nov 25, 2024
db9d424
test: add unit test for prune pipeline (segment pruner and block pruner)
dqhl76 Nov 25, 2024
3a7fccf
enable flag
dqhl76 Nov 26, 2024
7056d31
Merge branch 'main' into split2
dqhl76 Nov 26, 2024
c63a9af
Merge branch 'main' into split2
dqhl76 Nov 26, 2024
fdbf0f2
apply review suggestion
dqhl76 Nov 27, 2024
034f98f
Merge remote-tracking branch 'upstream/main' into split2
dqhl76 Nov 27, 2024
ff733b0
fmt
dqhl76 Nov 27, 2024
421f794
fix
dqhl76 Nov 27, 2024
0910d94
fix
dqhl76 Nov 27, 2024
58275f3
revert
dqhl76 Nov 27, 2024
c8e9076
Merge branch 'main' into split2
zhang2014 Nov 29, 2024
a743495
Merge branch 'main' into split2
zhang2014 Dec 8, 2024
b0efcc4
Merge branch 'main' into split2
zhang2014 Dec 9, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions src/query/catalog/src/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,17 @@ pub trait Table: Sync + Send {
)))
}

fn build_prune_pipeline(
&self,
table_ctx: Arc<dyn TableContext>,
plan: &DataSourcePlan,
source_pipeline: &mut Pipeline,
) -> Result<Option<Pipeline>> {
let (_, _, _) = (table_ctx, plan, source_pipeline);

Ok(None)
}

/// Assembly the pipeline of appending data to storage
fn append_data(&self, ctx: Arc<dyn TableContext>, pipeline: &mut Pipeline) -> Result<()> {
let (_, _) = (ctx, pipeline);
Expand Down
9 changes: 9 additions & 0 deletions src/query/service/src/pipelines/builders/builder_scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,15 @@ impl PipelineBuilder {
self.ctx.set_partitions(scan.source.parts.clone())?;
self.ctx
.set_wait_runtime_filter(scan.scan_id, self.contain_sink_processor);
if self.ctx.get_settings().get_enable_prune_pipeline()? {
if let Some(prune_pipeline) = table.build_prune_pipeline(
self.ctx.clone(),
&scan.source,
&mut self.main_pipeline,
)? {
self.pipelines.push(prune_pipeline);
}
}
table.read_data(
self.ctx.clone(),
&scan.source,
Expand Down
1 change: 1 addition & 0 deletions src/query/service/tests/it/storages/fuse/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ mod io;
mod meta;
mod operations;
mod pruning;
mod pruning_pipeline;
mod statistics;
mod table;
mod table_functions;
Expand Down
Loading
Loading