diff --git a/bindings/python/README.md b/bindings/python/README.md index 9a55cf8b7..6244c9439 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -158,6 +158,22 @@ print(f"\nRead: {batches_tt[0].num_rows} rows") print(batches_tt[0]) ``` +### Scan statistics + +Every scan collects lightweight counters during ordinary metadata planning: + +```python +plan = table.new_read_builder().new_scan().plan() +scan_stats = plan.stats() +print(scan_stats.manifest_entries_read, scan_stats.final_files) +``` + +Collection retains early manifest/sidecar pruning and LIMIT stopping. Counters +record the work performed; skipped entries are not decoded to reconstruct totals. +Combined incremental scans expose +the same API for their selected Delta or Changelog window. Snapshot metadata is +available from `plan.snapshot_id()`. + ### Shard column updates The batch and stream update objects expose diff --git a/bindings/python/python/pypaimon_rust/datafusion.pyi b/bindings/python/python/pypaimon_rust/datafusion.pyi index aa2829821..47fa04660 100644 --- a/bindings/python/python/pypaimon_rust/datafusion.pyi +++ b/bindings/python/python/pypaimon_rust/datafusion.pyi @@ -65,8 +65,68 @@ class Plan: """Selected snapshot, preserved for empty plans; None if no snapshot was selected.""" ... def splits(self) -> List[Split]: ... + def stats(self) -> ScanStats: + """Lightweight statistics collected during ordinary planning.""" + ... def __len__(self) -> int: ... +class ScanStats: + """Read-only counters from normal metadata planning.""" + @property + def base_manifest_files(self) -> int: ... + @property + def delta_manifest_files(self) -> int: ... + @property + def manifest_files_before_partition_pruning(self) -> int: ... + @property + def manifest_files_after_partition_pruning(self) -> int: ... + @property + def manifest_files_pruned_by_row_ranges(self) -> int: ... + @property + def manifest_entries_read(self) -> int: ... + @property + def manifest_entries_pruned_by_bucket(self) -> int: ... + @property + def manifest_entries_pruned_by_partition(self) -> int: ... + @property + def manifest_entries_after_entry_pruning(self) -> int: ... + @property + def manifest_entries_pruned_by_level(self) -> int: ... + @property + def manifest_entries_pruned_by_row_ranges(self) -> int: ... + @property + def manifest_entries_pruned_by_data_stats(self) -> int: ... + @property + def manifest_entries_after_manifest_filters(self) -> int: ... + @property + def manifest_entries_after_merge(self) -> int: ... + @property + def manifest_entries_pruned_by_cross_schema_stats(self) -> int: ... + @property + def manifest_entries_after_cross_schema_stats(self) -> int: ... + @property + def data_evolution_groups_before_stats(self) -> int: ... + @property + def data_evolution_groups_pruned_by_stats(self) -> int: ... + @property + def data_evolution_groups_pruned_by_row_ranges(self) -> int: ... + @property + def split_candidates_built(self) -> int: ... + @property + def limit_early_stopped(self) -> bool: ... + @property + def splits_before_limit(self) -> int: ... + @property + def splits_after_limit(self) -> int: ... + @property + def final_splits(self) -> int: ... + @property + def final_files(self) -> int: ... + @property + def planned_data_file_bytes(self) -> int: ... + @property + def limit(self) -> Optional[int]: ... + class VindexVectorGlobalIndexReader: """Rust index search over an input exposing pread_many(ranges).""" def __init__( diff --git a/bindings/python/src/context.rs b/bindings/python/src/context.rs index 2d9699094..8177a75b0 100644 --- a/bindings/python/src/context.rs +++ b/bindings/python/src/context.rs @@ -817,6 +817,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> this.add_class::()?; this.add_class::()?; this.add_class::()?; + this.add_class::()?; this.add_class::()?; this.add_class::()?; this.add_class::()?; diff --git a/bindings/python/src/read.rs b/bindings/python/src/read.rs index 8b7c167ba..587e712f9 100644 --- a/bindings/python/src/read.rs +++ b/bindings/python/src/read.rs @@ -758,6 +758,123 @@ impl PyTableScan { } } +#[pyclass(name = "ScanStats", module = "pypaimon_rust.datafusion", frozen)] +pub struct PyScanStats { + inner: paimon::table::ScanStats, +} + +#[pymethods] +impl PyScanStats { + #[getter] + fn base_manifest_files(&self) -> usize { + self.inner.base_manifest_files + } + #[getter] + fn delta_manifest_files(&self) -> usize { + self.inner.delta_manifest_files + } + #[getter] + fn manifest_files_before_partition_pruning(&self) -> usize { + self.inner.manifest_files_before_partition_pruning + } + #[getter] + fn manifest_files_after_partition_pruning(&self) -> usize { + self.inner.manifest_files_after_partition_pruning + } + #[getter] + fn manifest_files_pruned_by_row_ranges(&self) -> usize { + self.inner.manifest_files_pruned_by_row_ranges + } + #[getter] + fn manifest_entries_read(&self) -> usize { + self.inner.manifest_entries_read + } + #[getter] + fn manifest_entries_pruned_by_bucket(&self) -> usize { + self.inner.manifest_entries_pruned_by_bucket + } + #[getter] + fn manifest_entries_pruned_by_partition(&self) -> usize { + self.inner.manifest_entries_pruned_by_partition + } + #[getter] + fn manifest_entries_after_entry_pruning(&self) -> usize { + self.inner.manifest_entries_after_entry_pruning + } + #[getter] + fn manifest_entries_pruned_by_level(&self) -> usize { + self.inner.manifest_entries_pruned_by_level + } + #[getter] + fn manifest_entries_pruned_by_row_ranges(&self) -> usize { + self.inner.manifest_entries_pruned_by_row_ranges + } + #[getter] + fn manifest_entries_pruned_by_data_stats(&self) -> usize { + self.inner.manifest_entries_pruned_by_data_stats + } + #[getter] + fn manifest_entries_after_manifest_filters(&self) -> usize { + self.inner.manifest_entries_after_manifest_filters + } + #[getter] + fn manifest_entries_after_merge(&self) -> usize { + self.inner.manifest_entries_after_merge + } + #[getter] + fn manifest_entries_pruned_by_cross_schema_stats(&self) -> usize { + self.inner.manifest_entries_pruned_by_cross_schema_stats + } + #[getter] + fn manifest_entries_after_cross_schema_stats(&self) -> usize { + self.inner.manifest_entries_after_cross_schema_stats + } + #[getter] + fn data_evolution_groups_before_stats(&self) -> usize { + self.inner.data_evolution_groups_before_stats + } + #[getter] + fn data_evolution_groups_pruned_by_stats(&self) -> usize { + self.inner.data_evolution_groups_pruned_by_stats + } + #[getter] + fn data_evolution_groups_pruned_by_row_ranges(&self) -> usize { + self.inner.data_evolution_groups_pruned_by_row_ranges + } + #[getter] + fn split_candidates_built(&self) -> usize { + self.inner.split_candidates_built + } + #[getter] + fn limit_early_stopped(&self) -> bool { + self.inner.limit_early_stopped + } + #[getter] + fn splits_before_limit(&self) -> usize { + self.inner.splits_before_limit + } + #[getter] + fn splits_after_limit(&self) -> usize { + self.inner.splits_after_limit + } + #[getter] + fn final_splits(&self) -> usize { + self.inner.final_splits + } + #[getter] + fn final_files(&self) -> usize { + self.inner.final_files + } + #[getter] + fn planned_data_file_bytes(&self) -> u64 { + self.inner.planned_data_file_bytes + } + #[getter] + fn limit(&self) -> Option { + self.inner.limit + } +} + #[pyclass(name = "TableRead", module = "pypaimon_rust.datafusion")] pub struct PyTableRead { table: Arc, @@ -933,16 +1050,12 @@ impl PyRecordBatchReader { #[pyclass(name = "Plan", module = "pypaimon_rust.datafusion")] pub struct PyPlan { - splits: Vec, - snapshot_id: Option, + inner: paimon::table::Plan, } impl From for PyPlan { fn from(plan: paimon::table::Plan) -> Self { - Self { - splits: plan.splits().to_vec(), - snapshot_id: plan.snapshot_id(), - } + Self { inner: plan } } } @@ -951,11 +1064,18 @@ impl PyPlan { /// Snapshot selected by the scan, even when pruning produces no splits. /// `None` means no snapshot was selected, including snapshot-free format tables. fn snapshot_id(&self) -> Option { - self.snapshot_id + self.inner.snapshot_id() + } + + fn stats(&self) -> PyScanStats { + PyScanStats { + inner: self.inner.stats().clone(), + } } fn splits(&self) -> Vec { - self.splits + self.inner + .splits() .iter() .cloned() .map(|inner| PySplit { inner }) @@ -963,7 +1083,7 @@ impl PyPlan { } fn __len__(&self) -> usize { - self.splits.len() + self.inner.splits().len() } } diff --git a/bindings/python/tests/test_read.py b/bindings/python/tests/test_read.py index 4a47525d8..2183942af 100644 --- a/bindings/python/tests/test_read.py +++ b/bindings/python/tests/test_read.py @@ -51,6 +51,39 @@ def _make_variant_table_with_data(warehouse): return PaimonCatalog({"warehouse": warehouse}).get_table("vdb.t") +@pytest.mark.parametrize("limit,split_count", [(None, 1), (0, 0), (1, 1)]) +def test_scan_statistics_preserve_normal_planning(limit, split_count): + with tempfile.TemporaryDirectory() as warehouse: + table = _make_table_with_data(warehouse) + builder = table.new_read_builder() + if limit is not None: + builder.with_limit(limit) + scan = builder.new_scan() + plan = scan.plan() + stats = plan.stats() + assert len(plan.splits()) == split_count + assert plan.snapshot_id() == 1 + assert stats.manifest_entries_read == (0 if limit == 0 else 1) + assert stats.final_splits == stats.final_files == split_count + assert stats.limit == limit + if limit == 0: + assert stats.split_candidates_built == 0 + assert stats.limit_early_stopped + with pytest.raises(AttributeError): + stats.manifest_entries_read = 10 + + +@pytest.mark.parametrize("start,entry_count", [(0, 1), (1, 0)]) +def test_combined_incremental_scan_statistics(start, entry_count): + with tempfile.TemporaryDirectory() as warehouse: + table = _make_table_with_data(warehouse) + scan = table.new_read_builder().new_incremental_scan(start, 1) + plan = scan.plan() + stats = plan.stats() + assert plan.snapshot_id() == 1 + assert stats.manifest_entries_read == stats.final_files == entry_count + + def _variant_read_type(table, columns, paths, fail_on_error=False): fields = [] by_name = {field.name(): field for field in table.schema().fields()} diff --git a/bindings/python/tests/test_stream_table_scan.py b/bindings/python/tests/test_stream_table_scan.py index 2e81b69ac..079484286 100644 --- a/bindings/python/tests/test_stream_table_scan.py +++ b/bindings/python/tests/test_stream_table_scan.py @@ -62,6 +62,10 @@ def test_cursor_restore_and_initial_vs_follow_up(table, follow_up): plan = scan.plan() expected_id = 1 if follow_up else 2 assert plan.snapshot_id() == expected_id + stats = plan.stats() + assert stats.manifest_entries_read == stats.final_files == (4 if follow_up else 8) + assert stats.final_splits == len(plan.splits()) + assert stats.planned_data_file_bytes > 0 assert _rows(builder, plan) == [(i, (100 if follow_up else 200) + i) for i in range(32)] assert all(split.is_streaming() == follow_up for split in plan.splits()) assert scan.checkpoint() == expected_id + 1 diff --git a/crates/integration_tests/tests/read_tables.rs b/crates/integration_tests/tests/read_tables.rs index 5d58aeb18..7de3ea71d 100644 --- a/crates/integration_tests/tests/read_tables.rs +++ b/crates/integration_tests/tests/read_tables.rs @@ -27,7 +27,7 @@ use paimon::api::ConfigResponse; use paimon::catalog::{Identifier, RESTCatalog}; use paimon::common::Options; use paimon::spec::{DataType, IntType, Predicate, Schema, VarCharType}; -use paimon::{Catalog, CatalogOptions, Error, FileSystemCatalog, Plan, ScanTrace}; +use paimon::{Catalog, CatalogOptions, Error, FileSystemCatalog, Plan, ScanStats}; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, OnceLock}; @@ -1237,15 +1237,18 @@ async fn plan_table(table: &paimon::Table, limit: Option) -> Plan { scan.plan().await.expect("Failed to plan scan") } -async fn plan_table_with_trace(table: &paimon::Table, limit: Option) -> (Plan, ScanTrace) { +async fn plan_table_with_stats(table: &paimon::Table, limit: Option) -> (Plan, ScanStats) { let mut read_builder = table.new_read_builder(); if let Some(limit) = limit { read_builder.with_limit(limit); } let scan = read_builder.new_scan(); - scan.plan_with_trace() + let plan = scan + .plan() .await - .expect("Failed to plan scan with trace") + .expect("Failed to plan scan with statistics"); + let stats = plan.stats().clone(); + (plan, stats) } /// Test limit pushdown: when limit is smaller than total rows, fewer data files may be generated. @@ -1257,11 +1260,11 @@ async fn test_limit_pushdown() { let table = get_table_from_catalog(&catalog, "data_evolution_table").await; // Get full plan without limit - let (full_plan, full_trace) = plan_table_with_trace(&table, None).await; + let (full_plan, full_stats) = plan_table_with_stats(&table, None).await; let full_data_split_count: usize = full_plan.splits().iter().count(); // Get the plan with limit = 2 - let (limited_plan, limited_trace) = plan_table_with_trace(&table, Some(2)).await; + let (limited_plan, limited_stats) = plan_table_with_stats(&table, Some(2)).await; let limited_data_split_count: usize = limited_plan.splits().iter().count(); // For data evolution tables, limit pushdown at split level uses merged_row_count @@ -1271,12 +1274,12 @@ async fn test_limit_pushdown() { "Limit pushdown should reduce data split count for data evolution table: limited={limited_data_split_count}, full={full_data_split_count}" ); assert!( - limited_trace.limit_early_stopped, - "Limit pushdown should stop split construction early: {limited_trace:?}" + limited_stats.limit_early_stopped, + "Limit pushdown should stop split construction early: {limited_stats:?}" ); assert!( - limited_trace.split_candidates_built < full_trace.split_candidates_built, - "Limit pushdown should build fewer split candidates: limited={limited_trace:?}, full={full_trace:?}" + limited_stats.split_candidates_built < full_stats.split_candidates_built, + "Limit pushdown should build fewer split candidates: limited={limited_stats:?}, full={full_stats:?}" ); // Verify data evolution splits have merged_row_count. Overlapping row-id @@ -3356,20 +3359,22 @@ async fn test_limit_pushdown_disabled_with_row_ranges() { let mut row_range_reader = table.new_read_builder(); row_range_reader.with_row_ranges(vec![RowRange::new(0, i64::MAX)]); - let (row_range_plan, row_range_trace) = row_range_reader + let row_range_plan = row_range_reader .new_scan() - .plan_with_trace() + .plan() .await .expect("Failed to plan row-range scan"); + let row_range_stats = row_range_plan.stats().clone(); let mut limited_reader = table.new_read_builder(); limited_reader.with_row_ranges(vec![RowRange::new(0, i64::MAX)]); limited_reader.with_limit(2); - let (limited_plan, limited_trace) = limited_reader + let limited_plan = limited_reader .new_scan() - .plan_with_trace() + .plan() .await .expect("Failed to plan limited row-range scan"); + let limited_stats = limited_plan.stats().clone(); assert_eq!( limited_plan.splits().len(), @@ -3377,11 +3382,11 @@ async fn test_limit_pushdown_disabled_with_row_ranges() { "Positive limit pushdown should be disabled when effective row_ranges exist" ); assert!( - !limited_trace.limit_early_stopped, - "Row ranges should disable limit early stop: {limited_trace:?}" + !limited_stats.limit_early_stopped, + "Row ranges should disable limit early stop: {limited_stats:?}" ); assert_eq!( - limited_trace.split_candidates_built, row_range_trace.split_candidates_built, + limited_stats.split_candidates_built, row_range_stats.split_candidates_built, "Row-range limited scan should build the same split candidates as row-range scan" ); } diff --git a/crates/integrations/datafusion/src/full_text_search.rs b/crates/integrations/datafusion/src/full_text_search.rs index 9173e9fbe..4a4b4c577 100644 --- a/crates/integrations/datafusion/src/full_text_search.rs +++ b/crates/integrations/datafusion/src/full_text_search.rs @@ -186,7 +186,6 @@ impl TableProvider for FullTextSearchTableProvider { table, schema: &self.schema(), plan, - scan_trace: None, projection, pushed_predicate: None, limit, diff --git a/crates/integrations/datafusion/src/hybrid_search.rs b/crates/integrations/datafusion/src/hybrid_search.rs index fd8cddddb..314796031 100644 --- a/crates/integrations/datafusion/src/hybrid_search.rs +++ b/crates/integrations/datafusion/src/hybrid_search.rs @@ -293,7 +293,6 @@ impl TableProvider for HybridSearchTableProvider { table, schema: &input_schema, plan, - scan_trace: None, projection: Some(&input_projection), pushed_predicate: None, limit: None, diff --git a/crates/integrations/datafusion/src/physical_plan/audit_log.rs b/crates/integrations/datafusion/src/physical_plan/audit_log.rs index 072779b5b..50df0b1e6 100644 --- a/crates/integrations/datafusion/src/physical_plan/audit_log.rs +++ b/crates/integrations/datafusion/src/physical_plan/audit_log.rs @@ -140,7 +140,6 @@ mod tests { table: &table, schema: &arrow_schema, plan: paimon::table::Plan::new(vec![split(1), split(2)]), - scan_trace: None, projection: None, pushed_predicate: None, limit: None, diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index 0a9eb5ab6..e7edacbaa 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -53,7 +53,7 @@ use paimon::arrow::ReadBudget; use paimon::spec::{ CoreOptions, DataField, Datum, MergeEngine, Predicate, PredicateBuilder, PredicateOperator, }; -use paimon::table::{ArrowRecordBatchStream, ScanTrace, Table, TableRead}; +use paimon::table::{ArrowRecordBatchStream, ScanStats, Table, TableRead}; use paimon::DataSplit; use crate::error::to_datafusion_error; @@ -789,8 +789,8 @@ pub struct PaimonTableScan { /// Whether the pushed predicate is exact (no residual filtering needed). /// When true and all splits have known merged_row_count, statistics can be exact. filter_exact: bool, - /// Metadata-pruning trace captured during eager scan planning. - scan_trace: Option, + /// Metadata-pruning statistics captured during eager scan planning. + scan_stats: Option, /// Human-readable Variant extraction summary for explain output. pushed_variants: Option, /// Column-name case sensitivity carried from planning to execution so the @@ -818,7 +818,7 @@ impl PaimonTableScan { planned_partitions: Vec>, limit: Option, filter_exact: bool, - scan_trace: Option, + scan_stats: Option, pushed_variants: Option, case_sensitive: bool, ) -> Self { @@ -830,7 +830,7 @@ impl PaimonTableScan { planned_partitions, limit, filter_exact, - scan_trace, + scan_stats, pushed_variants, case_sensitive, Arc::new(ReadBudget::default()), @@ -846,7 +846,7 @@ impl PaimonTableScan { planned_partitions: Vec>, limit: Option, filter_exact: bool, - scan_trace: Option, + scan_stats: Option, pushed_variants: Option, case_sensitive: bool, ) -> DFResult { @@ -870,7 +870,7 @@ impl PaimonTableScan { planned_partitions, limit, filter_exact, - scan_trace, + scan_stats, pushed_variants, case_sensitive, parquet_read_budget, @@ -886,7 +886,7 @@ impl PaimonTableScan { planned_partitions: Vec>, limit: Option, filter_exact: bool, - scan_trace: Option, + scan_stats: Option, pushed_variants: Option, case_sensitive: bool, parquet_read_budget: Arc, @@ -905,7 +905,7 @@ impl PaimonTableScan { plan_properties, limit, filter_exact, - scan_trace, + scan_stats, pushed_variants, case_sensitive, runtime_filters: Vec::new(), @@ -1280,8 +1280,8 @@ impl DisplayAs for PaimonTableScan { if let Some(limit) = self.limit { write!(f, ", limit={limit}")?; } - if let Some(trace) = self.scan_trace.as_ref().filter(|_| !hidden) { - write!(f, ", trace={trace}")?; + if let Some(scan_stats) = self.scan_stats.as_ref().filter(|_| !hidden) { + write!(f, ", stats={scan_stats}")?; } if let Some(ref pushed_variants) = self.pushed_variants { write!(f, ", PushedVariants=[{pushed_variants}]")?; diff --git a/crates/integrations/datafusion/src/system_tables/audit_log.rs b/crates/integrations/datafusion/src/system_tables/audit_log.rs index b464afd4a..e19515a1a 100644 --- a/crates/integrations/datafusion/src/system_tables/audit_log.rs +++ b/crates/integrations/datafusion/src/system_tables/audit_log.rs @@ -95,7 +95,8 @@ impl TableProvider for AuditLogTable { if let Some(limit) = pushed_limit { read_builder.with_limit(limit); } - let (plan, trace) = await_with_runtime(read_builder.new_audit_scan().plan_with_trace()) + let scan = read_builder.new_audit_scan(); + let plan = await_with_runtime(scan.plan()) .await .map_err(to_datafusion_error)?; @@ -103,7 +104,6 @@ impl TableProvider for AuditLogTable { table: &self.table, schema: &self.schema, plan, - scan_trace: Some(trace), projection, pushed_predicate: filter_analysis.pushed_predicate, limit: pushed_limit, diff --git a/crates/integrations/datafusion/src/table/mod.rs b/crates/integrations/datafusion/src/table/mod.rs index 9a4727c5b..a3ba23252 100644 --- a/crates/integrations/datafusion/src/table/mod.rs +++ b/crates/integrations/datafusion/src/table/mod.rs @@ -359,7 +359,6 @@ pub(crate) struct PaimonScanBuilder<'a> { pub(crate) table: &'a Table, pub(crate) schema: &'a ArrowSchemaRef, pub(crate) plan: paimon::table::Plan, - pub(crate) scan_trace: Option, pub(crate) projection: Option<&'a Vec>, pub(crate) pushed_predicate: Option, pub(crate) limit: Option, @@ -400,6 +399,7 @@ impl PaimonScanBuilder<'_> { (self.schema.clone(), read_fields) }; + let scan_stats = Some(self.plan.stats().clone()); let splits = self.plan.into_splits(); let planned_partitions: Vec> = if splits.is_empty() { vec![Arc::from(Vec::new())] @@ -419,7 +419,7 @@ impl PaimonScanBuilder<'_> { planned_partitions, self.limit, self.filter_exact, - self.scan_trace, + scan_stats, None, self.case_sensitive, ) @@ -484,7 +484,7 @@ impl TableProvider for PaimonTableProvider { // Tokio runtime. `scan.plan()` can reach OpenDAL/Tokio filesystem calls while // reading Paimon metadata, so we must provide a runtime here instead of // assuming the caller already entered one. - let (plan, scan_trace) = await_with_runtime(scan.plan_with_trace()) + let plan = await_with_runtime(scan.plan()) .await .map_err(to_datafusion_error)?; @@ -498,7 +498,6 @@ impl TableProvider for PaimonTableProvider { table: &self.table, schema: &self.schema, plan, - scan_trace: Some(scan_trace), projection, pushed_predicate: filter_analysis.pushed_predicate, limit: pushed_limit, diff --git a/crates/integrations/datafusion/src/variant_pushdown.rs b/crates/integrations/datafusion/src/variant_pushdown.rs index 4955b302c..15ecfb403 100644 --- a/crates/integrations/datafusion/src/variant_pushdown.rs +++ b/crates/integrations/datafusion/src/variant_pushdown.rs @@ -227,10 +227,11 @@ impl ExtensionPlanner for VariantExtractionExtensionPlanner { read_builder.with_limit(limit); } let scan = read_builder.new_scan(); - let (plan, scan_trace) = await_with_runtime(scan.plan_with_trace()) + let plan = await_with_runtime(scan.plan()) .await .map_err(to_datafusion_error)?; + let scan_stats = Some(plan.stats().clone()); let splits = plan.into_splits(); let target = session_state.config_options().execution.target_partitions; let planned_partitions: Vec> = if splits.is_empty() { @@ -256,7 +257,7 @@ impl ExtensionPlanner for VariantExtractionExtensionPlanner { planned_partitions, pushed_limit, filter_exact, - Some(scan_trace), + scan_stats, Some(node.pushed_variants.clone()), case_sensitive, )?))) diff --git a/crates/integrations/datafusion/tests/query_auth.rs b/crates/integrations/datafusion/tests/query_auth.rs index a063f7ab7..31292e786 100644 --- a/crates/integrations/datafusion/tests/query_auth.rs +++ b/crates/integrations/datafusion/tests/query_auth.rs @@ -389,9 +389,9 @@ async fn test_query_auth_explain_shows_the_restriction_but_not_the_files() { assert!(!plan.contains("files="), "{plan}"); } -/// Pruning can leave no split to carry the grant; the counts and trace still precede the rules. +/// Pruning can leave no split to carry the grant; the counts and statistics still precede the rules. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_query_auth_explain_hides_the_trace_when_pruning_leaves_no_split() { +async fn test_query_auth_explain_hides_statistics_when_pruning_leaves_no_split() { let (_tmp, _server, context) = restricted_people(None).await; let plan = datafusion::arrow::util::pretty::pretty_format_batches( @@ -404,7 +404,7 @@ async fn test_query_auth_explain_hides_the_trace_when_pruning_leaves_no_split() .unwrap() .to_string(); assert!(!plan.contains("files="), "{plan}"); - assert!(!plan.contains("trace="), "{plan}"); + assert!(!plan.contains("stats="), "{plan}"); } /// Rows `(1, {"x":"bad"})` and `(2, {"x":7})`, served under the rule `id > 1`. diff --git a/crates/integrations/datafusion/tests/scan_pruning_trace.rs b/crates/integrations/datafusion/tests/scan_pruning_stats.rs similarity index 67% rename from crates/integrations/datafusion/tests/scan_pruning_trace.rs rename to crates/integrations/datafusion/tests/scan_pruning_stats.rs index 1b4465b6c..27929709a 100644 --- a/crates/integrations/datafusion/tests/scan_pruning_trace.rs +++ b/crates/integrations/datafusion/tests/scan_pruning_stats.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Self-contained baselines for scan-pruning trace counters. +//! Self-contained baselines for scan-pruning statistics counters. mod common; @@ -27,7 +27,7 @@ use paimon::catalog::Identifier; use paimon::spec::{Datum, PredicateBuilder}; use paimon::{Catalog, Table}; -async fn setup_trace_table() -> (tempfile::TempDir, std::sync::Arc) { +async fn setup_stats_table() -> (tempfile::TempDir, std::sync::Arc) { let (tmp, catalog) = common::create_test_env(); let sql_context = common::create_sql_context(catalog.clone()).await; sql_context @@ -39,7 +39,7 @@ async fn setup_trace_table() -> (tempfile::TempDir, std::sync::Arc (tempfile::TempDir, std::sync::Arc usize { +fn stats_counter(plan_text: &str, key: &str) -> usize { let needle = format!("{key}="); let start = plan_text .find(&needle) - .unwrap_or_else(|| panic!("trace counter {key} not found in plan:\n{plan_text}")) + .unwrap_or_else(|| panic!("statistics counter {key} not found in plan:\n{plan_text}")) + needle.len(); let end = plan_text[start..] .find(|c: char| !c.is_ascii_digit()) @@ -94,7 +94,7 @@ fn trace_counter(plan_text: &str, key: &str) -> usize { plan_text[start..end].parse().unwrap() } -fn trace_manifest_counts(plan_text: &str) -> (usize, usize) { +fn stats_manifest_counts(plan_text: &str) -> (usize, usize) { let needle = "manifests="; let start = plan_text .find(needle) @@ -117,7 +117,7 @@ fn trace_manifest_counts(plan_text: &str) -> (usize, usize) { /// one file holding a single distinct value, so `value NOT IN (20)` leaves the /// 20-only file with min == max == 20. #[tokio::test] -async fn test_scan_trace_records_not_in_data_stats_pruning() { +async fn test_scan_stats_records_not_in_data_stats_pruning() { let (tmp, catalog) = common::create_test_env(); let sql_context = common::create_sql_context(catalog.clone()).await; sql_context @@ -128,7 +128,7 @@ async fn test_scan_trace_records_not_in_data_stats_pruning() { .await .unwrap(); sql_context - .sql("CREATE TABLE paimon.test_db.trace_not_in (id INT, value INT)") + .sql("CREATE TABLE paimon.test_db.stats_not_in (id INT, value INT)") .await .unwrap() .collect() @@ -137,36 +137,33 @@ async fn test_scan_trace_records_not_in_data_stats_pruning() { for (id, value) in [(1, 10), (2, 20), (3, 30)] { common::exec( &sql_context, - &format!("INSERT INTO paimon.test_db.trace_not_in VALUES ({id}, {value})"), + &format!("INSERT INTO paimon.test_db.stats_not_in VALUES ({id}, {value})"), ) .await; } - let table = load_table(&catalog, "trace_not_in").await; + let table = load_table(&catalog, "stats_not_in").await; let fields = table.schema().fields(); let pb = PredicateBuilder::new(fields); - let (_plan, all_trace) = table - .new_read_builder() - .new_scan() - .plan_with_trace() - .await - .unwrap(); - assert_eq!(all_trace.final_files, 3); + let _plan = table.new_read_builder().new_scan().plan().await.unwrap(); + let all_stats = _plan.stats().clone(); + assert_eq!(all_stats.final_files, 3); let mut reader = table.new_read_builder(); reader.with_filter(pb.is_not_in("value", vec![Datum::Int(20)]).unwrap()); - let (_not_in_plan, not_in_trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let _not_in_plan = reader.new_scan().plan().await.unwrap(); + let not_in_stats = _not_in_plan.stats().clone(); assert_eq!( - not_in_trace.manifest_entries_pruned_by_data_stats, 1, - "the file holding only 20 should be pruned: {not_in_trace:?}" + not_in_stats.manifest_entries_pruned_by_data_stats, 1, + "the file holding only 20 should be pruned: {not_in_stats:?}" ); - assert_eq!(not_in_trace.final_files, 2); + assert_eq!(not_in_stats.final_files, 2); let rows = common::collect_id_value( &sql_context, - "SELECT id, value FROM paimon.test_db.trace_not_in WHERE value NOT IN (20) ORDER BY id", + "SELECT id, value FROM paimon.test_db.stats_not_in WHERE value NOT IN (20) ORDER BY id", ) .await; assert_eq!(rows, vec![(1, 10), (3, 30)]); @@ -174,18 +171,14 @@ async fn test_scan_trace_records_not_in_data_stats_pruning() { } #[tokio::test] -async fn test_scan_trace_records_partition_pruning() { - let (_tmp, catalog) = setup_trace_table().await; - let table = load_table(&catalog, "trace_append").await; +async fn test_scan_stats_records_partition_pruning() { + let (_tmp, catalog) = setup_stats_table().await; + let table = load_table(&catalog, "stats_append").await; - let (_all_plan, all_trace) = table - .new_read_builder() - .new_scan() - .plan_with_trace() - .await - .unwrap(); - assert_eq!(all_trace.snapshot_id, Some(3)); - assert_eq!(all_trace.final_files, 3); + let _all_plan = table.new_read_builder().new_scan().plan().await.unwrap(); + let all_stats = _all_plan.stats().clone(); + assert_eq!(_all_plan.snapshot_id(), Some(3)); + assert_eq!(all_stats.final_files, 3); let fields = table.schema().fields(); let pb = PredicateBuilder::new(fields); @@ -194,25 +187,25 @@ async fn test_scan_trace_records_partition_pruning() { pb.equal("dt", Datum::String("2024-01-01".to_string())) .unwrap(), ); - let (_partition_plan, partition_trace) = - partition_reader.new_scan().plan_with_trace().await.unwrap(); + let _partition_plan = partition_reader.new_scan().plan().await.unwrap(); + let partition_stats = _partition_plan.stats().clone(); assert!( - partition_trace.manifest_files_after_partition_pruning - < partition_trace.manifest_files_before_partition_pruning - || partition_trace.manifest_entries_pruned_by_partition > 0, - "partition predicate should prune at manifest or entry level: {partition_trace:?}" + partition_stats.manifest_files_after_partition_pruning + < partition_stats.manifest_files_before_partition_pruning + || partition_stats.manifest_entries_pruned_by_partition > 0, + "partition predicate should prune at manifest or entry level: {partition_stats:?}" ); assert!( - partition_trace.final_files < all_trace.final_files, - "partition pruning should reduce final files: all={all_trace:?}, filtered={partition_trace:?}" + partition_stats.final_files < all_stats.final_files, + "partition pruning should reduce final files: all={all_stats:?}, filtered={partition_stats:?}" ); } #[tokio::test] -async fn test_sql_between_records_partition_pruning_trace() { - let (_tmp, catalog) = setup_trace_table().await; +async fn test_sql_between_records_partition_pruning_stats() { + let (_tmp, catalog) = setup_stats_table().await; let sql_context = common::create_sql_context(catalog).await; - let sql = "SELECT id, value FROM paimon.test_db.trace_append + let sql = "SELECT id, value FROM paimon.test_db.stats_append WHERE dt BETWEEN '2024-01-01' AND '2024-01-02'"; let plan = sql_context @@ -223,8 +216,8 @@ async fn test_sql_between_records_partition_pruning_trace() { .await .unwrap(); let plan_text = displayable(plan.as_ref()).indent(true).to_string(); - let (manifests_after, manifests_before) = trace_manifest_counts(&plan_text); - let partition_pruned = trace_counter(&plan_text, "partition_pruned"); + let (manifests_after, manifests_before) = stats_manifest_counts(&plan_text); + let partition_pruned = stats_counter(&plan_text, "partition_pruned"); assert!( manifests_after < manifests_before || partition_pruned > 0, "SQL BETWEEN over partition column should prune at manifest or entry level:\n{plan_text}" @@ -235,10 +228,10 @@ async fn test_sql_between_records_partition_pruning_trace() { } #[tokio::test] -async fn test_count_star_uses_statistics_without_scan_trace() { - let (_tmp, catalog) = setup_trace_table().await; +async fn test_count_star_uses_statistics_without_scan_stats() { + let (_tmp, catalog) = setup_stats_table().await; let sql_context = common::create_sql_context(catalog).await; - let sql = "SELECT COUNT(*) FROM paimon.test_db.trace_append"; + let sql = "SELECT COUNT(*) FROM paimon.test_db.stats_append"; let df = sql_context.sql(sql).await.unwrap(); let plan = df.create_physical_plan().await.unwrap(); @@ -248,8 +241,8 @@ async fn test_count_star_uses_statistics_without_scan_trace() { "COUNT(*) should be satisfied from exact scan statistics without data scan:\n{plan_text}" ); assert!( - !plan_text.contains("trace="), - "COUNT(*) statistics rewrite should remove the scan node before trace display:\n{plan_text}" + !plan_text.contains("stats="), + "COUNT(*) statistics rewrite should remove the scan node before scan_stats display:\n{plan_text}" ); let batches = sql_context.sql(sql).await.unwrap().collect().await.unwrap(); @@ -263,13 +256,13 @@ async fn test_count_star_uses_statistics_without_scan_trace() { } #[tokio::test] -async fn test_scan_trace_records_bucket_limit_and_time_travel() { +async fn test_scan_stats_records_bucket_limit_and_time_travel() { let (tmp, catalog) = common::create_test_env(); let sql_context = common::create_sql_context(catalog.clone()).await; common::exec(&sql_context, "CREATE SCHEMA paimon.test_db").await; common::exec( &sql_context, - "CREATE TABLE paimon.test_db.trace_pk ( + "CREATE TABLE paimon.test_db.stats_pk ( id INT NOT NULL, value INT, PRIMARY KEY (id) ) WITH ('bucket' = '4')", @@ -277,79 +270,74 @@ async fn test_scan_trace_records_bucket_limit_and_time_travel() { .await; common::exec( &sql_context, - "INSERT INTO paimon.test_db.trace_pk VALUES + "INSERT INTO paimon.test_db.stats_pk VALUES (1, 10), (2, 20), (3, 30), (4, 40)", ) .await; common::exec( &sql_context, - "INSERT INTO paimon.test_db.trace_pk VALUES + "INSERT INTO paimon.test_db.stats_pk VALUES (5, 50), (6, 60), (7, 70), (8, 80)", ) .await; - let table = load_table(&catalog, "trace_pk").await; + let table = load_table(&catalog, "stats_pk").await; let fields = table.schema().fields(); let pb = PredicateBuilder::new(fields); let mut bucket_reader = table.new_read_builder(); bucket_reader.with_filter(pb.equal("id", Datum::Int(1)).unwrap()); - let (_bucket_plan, bucket_trace) = bucket_reader.new_scan().plan_with_trace().await.unwrap(); + let _bucket_plan = bucket_reader.new_scan().plan().await.unwrap(); + let bucket_stats = _bucket_plan.stats().clone(); assert!( - bucket_trace.manifest_entries_pruned_by_bucket > 0, - "bucket-key predicate should prune manifest entries by bucket: {bucket_trace:?}" + bucket_stats.manifest_entries_pruned_by_bucket > 0, + "bucket-key predicate should prune manifest entries by bucket: {bucket_stats:?}" ); - let (_full_plan, full_trace) = table - .new_read_builder() - .new_scan() - .plan_with_trace() - .await - .unwrap(); + let _full_plan = table.new_read_builder().new_scan().plan().await.unwrap(); + let full_stats = _full_plan.stats().clone(); let mut limit_reader = table.new_read_builder(); limit_reader.with_limit(1); - let (_limit_plan, limit_trace) = limit_reader.new_scan().plan_with_trace().await.unwrap(); + let _limit_plan = limit_reader.new_scan().plan().await.unwrap(); + let limit_stats = _limit_plan.stats().clone(); assert!( - limit_trace.limit_early_stopped, - "LIMIT should stop split construction early when no data residual exists: {limit_trace:?}" + limit_stats.limit_early_stopped, + "LIMIT should stop split construction early when no data residual exists: {limit_stats:?}" ); assert!( - limit_trace.split_candidates_built < full_trace.split_candidates_built, - "LIMIT should build fewer split candidates: limited={limit_trace:?}, full={full_trace:?}" + limit_stats.split_candidates_built < full_stats.split_candidates_built, + "LIMIT should build fewer split candidates: limited={limit_stats:?}, full={full_stats:?}" ); let snapshot_one_table = table.copy_with_options(HashMap::from([( "scan.version".to_string(), "1".to_string(), )])); - let (_snapshot_one_plan, snapshot_one_trace) = snapshot_one_table + let _snapshot_one_plan = snapshot_one_table .new_read_builder() .new_scan() - .plan_with_trace() + .plan() .await .unwrap(); - assert_eq!(snapshot_one_trace.snapshot_id, Some(1)); + let snapshot_one_stats = _snapshot_one_plan.stats().clone(); + assert_eq!(_snapshot_one_plan.snapshot_id(), Some(1)); - let (_latest_plan, latest_trace) = table - .new_read_builder() - .new_scan() - .plan_with_trace() - .await - .unwrap(); - assert_eq!(latest_trace.snapshot_id, Some(2)); + let _latest_plan = table.new_read_builder().new_scan().plan().await.unwrap(); + let latest_stats = _latest_plan.stats().clone(); + assert_eq!(_latest_plan.snapshot_id(), Some(2)); assert!( - snapshot_one_trace.final_files < latest_trace.final_files, - "time travel should plan from the selected snapshot: snapshot1={snapshot_one_trace:?}, latest={latest_trace:?}" + snapshot_one_stats.final_files < latest_stats.final_files, + "time travel should plan from the selected snapshot: snapshot1={snapshot_one_stats:?}, latest={latest_stats:?}" ); drop(tmp); } #[tokio::test] -async fn test_physical_plan_displays_scan_trace_summary() { - let (_tmp, catalog) = setup_trace_table().await; +async fn test_physical_plan_displays_scan_stats_summary() { + let (_tmp, catalog) = setup_stats_table().await; let sql_context = common::create_sql_context(catalog).await; let plan = sql_context - .sql("SELECT id FROM paimon.test_db.trace_append LIMIT 1") + .sql("SELECT id FROM paimon.test_db.stats_append LIMIT 1") .await .unwrap() .create_physical_plan() @@ -358,8 +346,8 @@ async fn test_physical_plan_displays_scan_trace_summary() { let plan_text = displayable(plan.as_ref()).indent(true).to_string(); assert!( - plan_text.contains("trace=") && plan_text.contains("splits_before_limit="), - "physical plan should include scan trace summary:\n{plan_text}" + plan_text.contains("stats=") && plan_text.contains("splits_before_limit="), + "physical plan should include scan scan_stats summary:\n{plan_text}" ); } @@ -383,7 +371,7 @@ async fn test_not_equal_keeps_float_files_with_nan_or_signed_zero() { .unwrap(); sql_context - .sql("CREATE TABLE paimon.test_db.trace_float (id INT, d DOUBLE)") + .sql("CREATE TABLE paimon.test_db.stats_float (id INT, d DOUBLE)") .await .unwrap() .collect() @@ -391,30 +379,30 @@ async fn test_not_equal_keeps_float_files_with_nan_or_signed_zero() { .unwrap(); common::exec( &sql_context, - "INSERT INTO paimon.test_db.trace_float VALUES \ + "INSERT INTO paimon.test_db.stats_float VALUES \ (1, CAST(1.0 AS DOUBLE)), (2, CAST('NaN' AS DOUBLE))", ) .await; common::exec( &sql_context, - "INSERT INTO paimon.test_db.trace_float VALUES (3, CAST(-0.0 AS DOUBLE))", + "INSERT INTO paimon.test_db.stats_float VALUES (3, CAST(-0.0 AS DOUBLE))", ) .await; for (sql, expected) in [ // The NaN row satisfies `<> 1.0`; pruning the file used to drop it. ( - "SELECT id FROM paimon.test_db.trace_float WHERE d <> 1.0 ORDER BY id", + "SELECT id FROM paimon.test_db.stats_float WHERE d <> 1.0 ORDER BY id", vec![2, 3], ), // `-0.0` is not `0.0` to the filter, so row 3 survives; rows 1 and 2 are // kept as well because `1.0 <> 0.0` and NaN is not `0.0` either. ( - "SELECT id FROM paimon.test_db.trace_float WHERE d <> 0.0 ORDER BY id", + "SELECT id FROM paimon.test_db.stats_float WHERE d <> 0.0 ORDER BY id", vec![1, 2, 3], ), ( - "SELECT id FROM paimon.test_db.trace_float WHERE d NOT IN (1.0, 2.0) ORDER BY id", + "SELECT id FROM paimon.test_db.stats_float WHERE d NOT IN (1.0, 2.0) ORDER BY id", vec![2, 3], ), ] { diff --git a/crates/paimon/src/lib.rs b/crates/paimon/src/lib.rs index 146c5e2b1..ead1c0759 100644 --- a/crates/paimon/src/lib.rs +++ b/crates/paimon/src/lib.rs @@ -54,7 +54,7 @@ pub use table::{ DataSplit, DataSplitBuilder, DeletionFile, IncrementalPlan, IncrementalScan, IncrementalScanMode, IncrementalSplit, PartitionBucket, Plan, PostponeBucketPlan, PostponeFixedBucketTableCommit, PostponeFixedBucketTableWrite, RESTEnv, RESTSnapshotCommit, - ReadBuilder, RenamingSnapshotCommit, RowRange, ScanTrace, SnapshotCommit, SnapshotManager, + ReadBuilder, RenamingSnapshotCommit, RowRange, ScanStats, SnapshotCommit, SnapshotManager, Table, TableCommit, TableRead, TableScan, TableUpdate, TableUpdateByRowId, TableWrite, TagManager, WriteBuilder, }; diff --git a/crates/paimon/src/table/audit_log_table/scan.rs b/crates/paimon/src/table/audit_log_table/scan.rs index fa14c0f09..6abca225c 100644 --- a/crates/paimon/src/table/audit_log_table/scan.rs +++ b/crates/paimon/src/table/audit_log_table/scan.rs @@ -21,7 +21,7 @@ use super::{TableScan, TableScanKind}; use crate::spec::MergeEngine; use crate::table::kv_file_reader::retain_primary_key_conjuncts; use crate::table::merge_tree_split_generator::{merge_tree_split_for_batch, KeyComparator}; -use crate::table::{DataSplit, Plan, ReadBuilder, ScanTrace}; +use crate::table::{DataSplit, Plan, ReadBuilder}; use std::collections::HashMap; /// Plans current-state audit reads over the wrapped table's files. @@ -50,91 +50,76 @@ impl<'a> AuditLogScan<'a> { } pub async fn plan(&self) -> crate::Result { - self.plan_with_trace().await.map(|(plan, _)| plan) - } - - pub async fn plan_with_trace(&self) -> crate::Result<(Plan, ScanTrace)> { let TableScanKind::Paimon(inner) = &self.scan.0 else { return Err(crate::Error::Unsupported { message: "Format tables do not support audit log batch scan".to_string(), }); }; - let (plan, mut trace) = self.scan.plan_with_trace().await?; + let mut plan = self.scan.plan().await?; let options = inner.table.schema().core_options(); let engine = options.merge_engine()?; // Ordinary first-row and DV scans pack files without a key merge. Audit // reads need the existing merge-tree planner to retain overlapping versions. - let plan = if engine == MergeEngine::FirstRow + if engine == MergeEngine::FirstRow || (options.deletion_vectors_enabled() && !options.deletion_vectors_merge_on_read()) { if let Some(comparator) = KeyComparator::from_table_schema(inner.table.schema()) { - let mut buckets: HashMap<(Vec, i32), Vec> = HashMap::new(); - for split in plan.into_splits() { - buckets - .entry((split.partition().to_serialized_bytes(), split.bucket())) - .or_default() - .push(split); - } - let mut splits = Vec::new(); - for bucket in buckets.into_values() { - let first = &bucket[0]; - let files = bucket - .iter() - .flat_map(|split| split.data_files().iter().cloned()) - .collect(); - let deletion_files: HashMap<_, _> = bucket - .iter() - .flat_map(|split| { - split.data_files().iter().filter_map(move |file| { - split - .deletion_file_for_data_file(file) - .map(|deletion| (file.file_name.clone(), deletion.clone())) + plan = plan.map_splits(|planned_splits| { + let mut buckets: HashMap<(Vec, i32), Vec> = HashMap::new(); + for split in planned_splits { + buckets + .entry((split.partition().to_serialized_bytes(), split.bucket())) + .or_default() + .push(split); + } + let mut splits = Vec::new(); + for bucket in buckets.into_values() { + let first = &bucket[0]; + let files = bucket + .iter() + .flat_map(|split| split.data_files().iter().cloned()) + .collect(); + let deletion_files: HashMap<_, _> = bucket + .iter() + .flat_map(|split| { + split.data_files().iter().filter_map(move |file| { + split + .deletion_file_for_data_file(file) + .map(|deletion| (file.file_name.clone(), deletion.clone())) + }) }) - }) - .collect(); - for group in merge_tree_split_for_batch( - files, - &comparator, - options.source_split_target_size(), - options.source_split_open_file_cost(), - matches!(engine, MergeEngine::Deduplicate | MergeEngine::FirstRow), - ) { - let mut builder = DataSplit::builder() - .with_snapshot(first.snapshot_id()) - .with_partition(first.partition().clone()) - .with_bucket(first.bucket()) - .with_bucket_path(first.bucket_path().to_string()) - .with_total_buckets(first.total_buckets()) - .with_raw_convertible(group.raw_convertible); - if !deletion_files.is_empty() { - builder = builder.with_data_deletion_files( - group - .files - .iter() - .map(|file| deletion_files.get(&file.file_name).cloned()) - .collect(), - ); + .collect(); + for group in merge_tree_split_for_batch( + files, + &comparator, + options.source_split_target_size(), + options.source_split_open_file_cost(), + matches!(engine, MergeEngine::Deduplicate | MergeEngine::FirstRow), + ) { + let mut builder = DataSplit::builder() + .with_snapshot(first.snapshot_id()) + .with_partition(first.partition().clone()) + .with_bucket(first.bucket()) + .with_bucket_path(first.bucket_path().to_string()) + .with_total_buckets(first.total_buckets()) + .with_raw_convertible(group.raw_convertible); + if !deletion_files.is_empty() { + builder = builder.with_data_deletion_files( + group + .files + .iter() + .map(|file| deletion_files.get(&file.file_name).cloned()) + .collect(), + ); + } + splits.push(builder.with_data_files(group.files).build()?); } - splits.push(builder.with_data_files(group.files).build()?); } - } - Plan::new(splits) - } else { - plan + Ok(splits) + })?; } - } else { - plan - }; - trace.record_final_plan( - plan.splits().len(), - plan.splits().len(), - plan.splits() - .iter() - .map(|split| split.data_files().len()) - .sum(), - ); - trace.planned_data_file_bytes = plan.planned_data_file_bytes(); - Ok((plan, trace)) + } + Ok(plan) } } @@ -153,7 +138,7 @@ mod tests { }; use crate::table::table_scan::{ tests::{ - data_evolution_test_table, pk_stats_file, pk_stats_gate_table, setup_scan_trace_dirs, + data_evolution_test_table, pk_stats_file, pk_stats_gate_table, setup_scan_stats_dirs, two_column_schema, }, PaimonTableScan, @@ -204,7 +189,7 @@ mod tests { (option.to_string(), value.to_string()), ("source.split.target-size".to_string(), "1b".to_string()), ])); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; let mut old = pk_stats_file("old-version.parquet", (1, 5), (100, 200)); old.level = 1; @@ -230,9 +215,9 @@ mod tests { let mut reader = table.new_read_builder(); reader.with_filter(value_filter); - let (ordinary_plan, ordinary_trace) = - reader.new_scan().plan_with_trace().await.unwrap(); - assert!(ordinary_trace.manifest_entries_pruned_by_data_stats >= 1); + let ordinary_plan = reader.new_scan().plan().await.unwrap(); + let ordinary_stats = ordinary_plan.stats().clone(); + assert!(ordinary_stats.manifest_entries_pruned_by_data_stats >= 1); assert_eq!( ordinary_plan .splits() @@ -242,9 +227,15 @@ mod tests { 1 ); - let (audit_plan, audit_trace) = - reader.new_audit_scan().plan_with_trace().await.unwrap(); - assert_eq!(audit_trace.manifest_entries_pruned_by_data_stats, 0); + let audit_plan = reader.new_audit_scan().plan().await.unwrap(); + let audit_stats = audit_plan.stats().clone(); + assert_eq!(audit_plan.snapshot_id(), ordinary_plan.snapshot_id()); + assert_eq!(audit_stats.final_files, 2); + assert_eq!( + audit_stats.planned_data_file_bytes, + audit_plan.planned_data_file_bytes() + ); + assert_eq!(audit_stats.manifest_entries_pruned_by_data_stats, 0); assert_eq!( audit_plan .splits() diff --git a/crates/paimon/src/table/bucket_filter.rs b/crates/paimon/src/table/bucket_filter.rs index 238cda431..c95d42c09 100644 --- a/crates/paimon/src/table/bucket_filter.rs +++ b/crates/paimon/src/table/bucket_filter.rs @@ -18,11 +18,203 @@ //! Bucket and partition predicate extraction and bucket hash pruning. use crate::spec::{ - field_idx_to_partition_idx, BucketFunctionType, DataField, DataType, Datum, Predicate, - PredicateOperator, + eval_row, field_idx_to_partition_idx, BinaryRow, BucketFunctionType, CoreOptions, DataField, + DataType, Datum, Predicate, PredicateOperator, }; use crate::table::bucket_function::bucket_for_datums; -use std::collections::HashSet; +use crate::table::Table; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +const MAX_VALUES: usize = 1000; + +/// Java's BucketSelector keeps a separate selector for each concrete partition. +/// Clone the immutable configuration for concurrent manifest tasks; each task +/// caches partition simplification and bucket counts without a shared lock. +#[derive(Clone)] +pub(super) struct BucketSelector { + config: Arc, + partitions: HashMap, PartitionSelector>, +} + +struct BucketSelectorConfig { + predicate: Predicate, + manifest_predicate: Option, + partition_mapping: Vec>, + bucket_mapping: Vec>, + partition_arity: usize, + partition_dependent: bool, + bucket_fields: Vec, + function: BucketFunctionType, +} + +#[derive(Clone)] +struct PartitionSelector { + predicate: Option, + buckets: HashMap>>, +} + +impl BucketSelector { + pub(super) fn new(table: &Table, predicate: &Predicate) -> crate::Result> { + let options = CoreOptions::new(table.schema().options()); + // Dynamic and cross-partition bucket assignments are index-driven, + // rather than a fixed hash modulo the file's bucket count. + if options.bucket() < 1 && options.bucket() != -2 { + return Ok(None); + } + let keys = options + .bucket_key() + .unwrap_or_else(|| table.schema().trimmed_primary_keys()); + let fields = table.schema().fields(); + let bucket_fields = keys + .iter() + .filter_map(|name| fields.iter().find(|field| field.name() == name).cloned()) + .collect::>(); + if keys.is_empty() || bucket_fields.len() != keys.len() { + return Ok(None); + } + let bucket_mapping = field_idx_to_partition_idx(fields, &keys); + let partition_mapping = field_idx_to_partition_idx(fields, table.schema().partition_keys()); + let mut referenced = HashSet::new(); + predicate.collect_leaf_field_indices(&mut referenced); + let partition_dependent = partition_mapping + .iter() + .enumerate() + .any(|(index, mapped)| mapped.is_some() && referenced.contains(&index)); + let config = BucketSelectorConfig { + predicate: predicate.clone(), + manifest_predicate: project_bucket_predicate(predicate, &bucket_mapping), + partition_mapping, + bucket_mapping, + partition_arity: table.schema().partition_keys().len(), + partition_dependent, + bucket_fields, + function: options.bucket_function_type()?, + }; + Ok(Some(Self { + config: Arc::new(config), + partitions: HashMap::new(), + })) + } + + pub(super) fn manifest_predicate(&self) -> Option<&Predicate> { + self.config.manifest_predicate.as_ref() + } + + pub(super) fn bucket_fields(&self) -> &[DataField] { + &self.config.bucket_fields + } + + pub(super) fn function(&self) -> BucketFunctionType { + self.config.function + } + + pub(super) fn test(&mut self, partition: &[u8], bucket: i32, total_buckets: i32) -> bool { + // Pending buckets carry rows with no hash assignment yet. Java keeps + // them independently of whether this scan excludes pending buckets. + if bucket < 0 || total_buckets <= 0 { + return true; + } + let config = &self.config; + // A key-only predicate has the same selector in every partition. + let partition = if config.partition_dependent { + partition + } else { + &[] + }; + if !self.partitions.contains_key(partition) { + let predicate = if !config.partition_dependent { + config.manifest_predicate.clone() + } else { + BinaryRow::from_serialized_bytes(partition) + .ok() + .filter(|row| row.arity() as usize == config.partition_arity) + .and_then(|row| { + replace_partition_predicates( + &config.predicate, + &config.partition_mapping, + &row, + ) + .ok() + }) + .map(|predicate| project_bucket_predicate(&predicate, &config.bucket_mapping)) + .unwrap_or_else(|| config.manifest_predicate.clone()) + }; + self.partitions.insert( + partition.to_vec(), + PartitionSelector { + predicate, + buckets: HashMap::new(), + }, + ); + } + let selector = self.partitions.get_mut(partition).unwrap(); + selector + .buckets + .entry(total_buckets) + .or_insert_with(|| config.targets(selector.predicate.as_ref(), total_buckets)) + .as_ref() + .is_none_or(|targets| targets.contains(&bucket)) + } +} + +impl BucketSelectorConfig { + fn targets(&self, predicate: Option<&Predicate>, total_buckets: i32) -> Option> { + compute_target_buckets( + predicate?, + &self.bucket_fields, + self.function, + total_buckets, + ) + } +} + +fn replace_partition_predicates( + predicate: &Predicate, + mapping: &[Option], + row: &BinaryRow, +) -> crate::Result { + Ok(match predicate { + Predicate::And(children) => Predicate::and( + children + .iter() + .map(|child| replace_partition_predicates(child, mapping, row)) + .collect::>()?, + ), + Predicate::Or(children) => Predicate::or( + children + .iter() + .map(|child| replace_partition_predicates(child, mapping, row)) + .collect::>()?, + ), + Predicate::Not(inner) => { + Predicate::negate(replace_partition_predicates(inner, mapping, row)?) + } + Predicate::Leaf { .. } => match predicate.remap_field_index(mapping) { + Some(partition) => { + if eval_row(&partition, row)? { + Predicate::AlwaysTrue + } else { + Predicate::AlwaysFalse + } + } + None => predicate.clone(), + }, + _ => predicate.clone(), + }) +} + +// Java pickTransformFieldMapping keeps complete top-level conjuncts. A mixed +// OR must never contribute just one of its branches to bucket selection. +fn project_bucket_predicate(predicate: &Predicate, mapping: &[Option]) -> Option { + let projected = predicate + .clone() + .split_and() + .iter() + .filter_map(|conjunct| conjunct.remap_field_index(mapping)) + .collect::>(); + (!projected.is_empty()).then(|| Predicate::and(projected)) +} pub(super) fn split_partition_and_data_predicates( filter: Predicate, @@ -104,18 +296,29 @@ pub(super) fn compute_target_buckets( } // Collect equal-value candidates per bucket key field (by projected index). - // Each field can have one value (Eq), multiple values (In), or NULL (IsNull). + // Java accepts ORs of Equal/In leaves on one key, intersecting repeated + // AND constraints. Other conjuncts do not help select buckets. let num_keys = bucket_key_fields.len(); - let mut field_candidates: Vec>>> = vec![None; num_keys]; + let mut field_candidates: Vec>> = vec![None; num_keys]; - collect_eq_candidates(bucket_predicate, &mut field_candidates); + collect_and_candidates(bucket_predicate, &mut field_candidates)?; // All bucket key fields must have candidates. - let candidates: Vec<&Vec>> = + let candidates: Vec<&Vec<&Datum>> = field_candidates.iter().filter_map(|c| c.as_ref()).collect(); if candidates.len() != num_keys { return None; } + let mut combinations = 1usize; + for values in &candidates { + combinations = combinations.checked_mul(values.len())?; + if combinations > MAX_VALUES { + return None; + } + } + if combinations == 0 { + return Some(HashSet::new()); + } // Compute cartesian product of candidates and hash each combination. let mut buckets = HashSet::new(); @@ -124,7 +327,7 @@ pub(super) fn compute_target_buckets( let datums: Vec<(Option<&Datum>, &DataType)> = (0..num_keys) .map(|i| { let vals = field_candidates[i].as_ref().unwrap(); - (vals[combo[i]], bucket_key_fields[i].data_type()) + (Some(vals[combo[i]]), bucket_key_fields[i].data_type()) }) .collect(); @@ -155,36 +358,331 @@ pub(super) fn compute_target_buckets( } } -/// Recursively collect Eq/In/IsNull literal candidates from a predicate for each bucket key field. -fn collect_eq_candidates<'a>( +fn collect_and_candidates<'a>( predicate: &'a Predicate, - field_candidates: &mut Vec>>>, -) { + candidates: &mut [Option>], +) -> Option<()> { + if let Predicate::And(children) = predicate { + for child in children { + collect_and_candidates(child, candidates)?; + } + } else if let Some((index, values)) = collect_or_candidates(predicate, candidates.len()) { + let mut values = values; + if let Some(previous) = &mut candidates[index] { + // Preserve the first conjunct's order and duplicates, as retainAll + // does in Java, without quadratic comparisons for large IN lists. + if let Some(first) = values.first().or(previous.first()) { + // Variant equality is defined for identical payloads, but + // distinct payloads have no ordering for sorting/searching. + if matches!(first, Datum::Variant { .. }) { + return None; + } + let kind = std::mem::discriminant(*first); + if values.iter().chain(previous.iter()).any(|value| { + std::mem::discriminant(*value) != kind || value.partial_cmp(value).is_none() + }) { + return None; + } + } + values.sort_unstable_by(|left, right| left.partial_cmp(right).unwrap()); + previous.retain(|value| { + values + .binary_search_by(|candidate| candidate.partial_cmp(value).unwrap()) + .is_ok() + }); + // Java fails open rather than using a contradictory candidate set. + if previous.is_empty() { + return None; + } + } else { + candidates[index] = Some(values); + } + } + Some(()) +} + +/// One OR clause is usable only when every branch is Equal/In on the same key. +fn collect_or_candidates(predicate: &Predicate, num_keys: usize) -> Option<(usize, Vec<&Datum>)> { match predicate { - Predicate::And(children) => { + Predicate::Or(children) => { + let mut reference = None; + let mut values = Vec::new(); for child in children { - collect_eq_candidates(child, field_candidates); + let (index, child_values) = collect_or_candidates(child, num_keys)?; + if reference.is_some_and(|previous| previous != index) { + return None; + } + reference = Some(index); + values.extend(child_values); } + Some((reference?, values)) } Predicate::Leaf { index, op, literals, .. - } if *index < field_candidates.len() => match op { - PredicateOperator::Eq => { - if let Some(lit) = literals.first() { - field_candidates[*index] = Some(vec![Some(lit)]); - } + } if *index < num_keys => match op { + PredicateOperator::Eq => Some((*index, vec![literals.first()?])), + PredicateOperator::In => Some((*index, literals.iter().collect())), + _ => None, + }, + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::Identifier; + use crate::io::FileIOBuilder; + use crate::spec::{ + BinaryRowBuilder, IntType, PredicateBuilder, Schema, TableSchema, VarCharType, + }; + + fn fields() -> Vec { + vec![ + DataField::new(0, "a".into(), DataType::Int(IntType::new())), + DataField::new(1, "b".into(), DataType::Int(IntType::new())), + ] + } + + fn buckets(predicate: &Predicate, fields: &[DataField]) -> Option> { + compute_target_buckets(predicate, fields, BucketFunctionType::Default, 128) + } + + #[test] + fn same_key_or_and_repeated_constraints_follow_java() { + let fields = fields(); + let pb = PredicateBuilder::new(&fields); + let a = pb.equal("a", Datum::Int(2)).unwrap(); + let b = pb.equal("a", Datum::Int(3)).unwrap(); + let or = Predicate::or(vec![a.clone(), b.clone()]); + let expected = buckets(&a, &fields[..1]) + .unwrap() + .union(&buckets(&b, &fields[..1]).unwrap()) + .copied() + .collect(); + assert_eq!(buckets(&or, &fields[..1]), Some(expected)); + let intersection = Predicate::and(vec![ + pb.is_in("a", vec![Datum::Int(1), Datum::Int(2)]).unwrap(), + or.clone(), + ]); + assert_eq!( + buckets(&intersection, &fields[..1]), + buckets(&a, &fields[..1]) + ); + let reversed = Predicate::and(vec![ + or, + pb.is_in("a", vec![Datum::Int(1), Datum::Int(2)]).unwrap(), + ]); + assert_eq!(buckets(&reversed, &fields[..1]), buckets(&a, &fields[..1])); + assert!(buckets(&Predicate::and(vec![a, b]), &fields[..1]).is_none()); + } + + #[test] + fn mixed_or_never_contributes_a_partial_key_constraint() { + let fields = fields(); + let pb = PredicateBuilder::new(&fields); + let key = pb.equal("a", Datum::Int(1)).unwrap(); + for unrelated in [ + pb.equal("b", Datum::Int(2)).unwrap(), + pb.greater_than("a", Datum::Int(10)).unwrap(), + ] { + let mixed = Predicate::or(vec![key.clone(), unrelated]); + assert!(buckets(&mixed, &fields[..1]).is_none()); + let safe = Predicate::and(vec![key.clone(), mixed]); + assert_eq!(buckets(&safe, &fields[..1]), buckets(&key, &fields[..1])); + } + let null = pb.is_null("a").unwrap(); + assert!(buckets(&null, &fields[..1]).is_none()); + } + + #[test] + fn candidate_product_is_bounded_before_hashing() { + let fields = fields(); + let pb = PredicateBuilder::new(&fields); + let first = pb.is_in("a", (0..40).map(Datum::Int).collect()).unwrap(); + let at_limit = Predicate::and(vec![ + first.clone(), + pb.is_in("b", (0..25).map(Datum::Int).collect()).unwrap(), + ]); + assert!(buckets(&at_limit, &fields).is_some()); + let above_limit = Predicate::and(vec![ + first, + pb.is_in("b", (0..26).map(Datum::Int).collect()).unwrap(), + ]); + assert!(buckets(&above_limit, &fields).is_none()); + // A large IN can still be narrowed by another conjunct before the cap. + let narrowed = Predicate::and(vec![ + pb.is_in("a", (0..1001).map(Datum::Int).collect()).unwrap(), + pb.equal("a", Datum::Int(3)).unwrap(), + ]); + assert_eq!( + buckets(&narrowed, &fields[..1]), + buckets(&pb.equal("a", Datum::Int(3)).unwrap(), &fields[..1]) + ); + } + + #[test] + fn large_intersections_are_reduced_before_the_combination_limit() { + let fields = fields(); + let pb = PredicateBuilder::new(&fields); + let left = pb + .is_in("a", (0..50_000).map(Datum::Int).collect()) + .unwrap(); + let right = pb + .is_in("a", (49_990..100_000).map(Datum::Int).collect()) + .unwrap(); + let expected = buckets( + &pb.is_in("a", (49_990..50_000).map(Datum::Int).collect()) + .unwrap(), + &fields[..1], + ); + assert!(expected.is_some()); + assert_eq!( + buckets( + &Predicate::and(vec![left.clone(), right.clone()]), + &fields[..1] + ), + expected + ); + assert_eq!( + buckets(&Predicate::and(vec![right, left]), &fields[..1]), + expected + ); + } + + #[test] + fn unordered_literals_in_repeated_constraints_fail_open() { + let make_leaf = |json: &str| { + let value = crate::variant::GenericVariant::parse_json(json).unwrap(); + Predicate::Leaf { + column: "v".into(), + index: 0, + data_type: DataType::Variant(crate::spec::VariantType::new()), + op: PredicateOperator::Eq, + literals: vec![Datum::Variant { + value: value.value().to_vec(), + metadata: value.metadata().to_vec(), + }], } - PredicateOperator::In if !literals.is_empty() => { - field_candidates[*index] = Some(literals.iter().map(Some).collect()); + }; + let predicate = Predicate::and(vec![make_leaf("1"), make_leaf("2")]); + let mut candidates = vec![None]; + assert!(collect_and_candidates(&predicate, &mut candidates).is_none()); + } + + fn partitioned_table(bucket: &str, function: &str) -> Table { + let schema = Schema::builder() + .column("pt", DataType::VarChar(VarCharType::default())) + .column("a", DataType::Int(IntType::new())) + .column("b", DataType::Int(IntType::new())) + .partition_keys(["pt"]) + .primary_key(["pt", "a", "b"]) + .option("bucket", bucket) + .option("bucket-key", "a,b") + .option("bucket-function.type", function) + .build() + .unwrap(); + Table::new( + FileIOBuilder::new("memory").build().unwrap(), + Identifier::new("db", "bucket"), + "memory:/buckets".into(), + TableSchema::new(0, &schema), + None, + ) + } + + fn partition(value: Option<&str>) -> Vec { + let mut row = BinaryRowBuilder::new(1); + match value { + Some(value) => row.write_string(0, value), + None => row.set_null_at(0), + } + row.build_serialized() + } + + #[test] + fn partition_specialization_and_bucket_count_caches_are_independent() { + for function in ["default", "hive"] { + let table = partitioned_table("8", function); + let pb = PredicateBuilder::new(table.schema().fields()); + let branch = |pt: &str, a: i32| { + Predicate::and(vec![ + pb.equal("pt", Datum::String(pt.into())).unwrap(), + pb.equal("a", Datum::Int(a)).unwrap(), + pb.is_in("b", vec![Datum::Int(1), Datum::Int(2)]).unwrap(), + ]) + }; + let predicate = Predicate::or(vec![branch("left", 10), branch("right", 20)]); + let mut selector = BucketSelector::new(&table, &predicate).unwrap().unwrap(); + // Before reading a partition, correlated ANDs inside OR cannot be + // interpreted as an OR on one bucket key, so early pruning stays open. + assert!(selector + .config + .targets(selector.manifest_predicate(), 8) + .is_none()); + for total in [8, 17, 8] { + for (pt, a) in [("left", 10), ("right", 20)] { + let expected = (1..=2) + .map(|b| { + bucket_for_datums( + &[ + ( + Some(&Datum::Int(a)), + selector.bucket_fields()[0].data_type(), + ), + ( + Some(&Datum::Int(b)), + selector.bucket_fields()[1].data_type(), + ), + ], + selector.function(), + total, + ) + .unwrap() + }) + .collect::>(); + let actual = (0..total) + .filter(|&bucket| selector.test(&partition(Some(pt)), bucket, total)) + .collect::>(); + assert_eq!(actual, expected); + } } - PredicateOperator::IsNull => { - field_candidates[*index] = Some(vec![None]); + // Java leaves an unmatchable partition to the partition filter. + for bucket in 0..8 { + assert!(selector.test(&partition(Some("missing")), bucket, 8)); + assert!(selector.test(&partition(None), bucket, 8)); + assert!(selector.test(b"invalid", bucket, 8)); } - _ => {} - }, - _ => {} + } + } + + #[test] + fn unconstrained_partition_branch_pending_buckets_and_dynamic_modes_fail_open() { + let table = partitioned_table("8", "default"); + let pb = PredicateBuilder::new(table.schema().fields()); + let predicate = Predicate::or(vec![ + Predicate::and(vec![ + pb.equal("pt", Datum::String("left".into())).unwrap(), + pb.equal("a", Datum::Int(1)).unwrap(), + pb.equal("b", Datum::Int(2)).unwrap(), + ]), + pb.equal("pt", Datum::String("right".into())).unwrap(), + ]); + let mut selector = BucketSelector::new(&table, &predicate).unwrap().unwrap(); + for bucket in 0..8 { + assert!(selector.test(&partition(Some("right")), bucket, 8)); + assert!(selector.test(&partition(Some("left")), bucket, 0)); + } + assert!(selector.test(&partition(Some("left")), -1, 8)); + assert!(selector.test(&partition(Some("left")), -2, 8)); + let dynamic = partitioned_table("-1", "default"); + assert!(BucketSelector::new(&dynamic, &predicate).unwrap().is_none()); + let postpone = partitioned_table("-2", "default"); + assert!(BucketSelector::new(&postpone, &predicate) + .unwrap() + .is_some()); } } diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 411bcb575..6fac24c65 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -7115,7 +7115,8 @@ mod tests { let mut builder = table.new_read_builder(); builder.with_row_ranges(vec![RowRange::new(0, 0)]); - let (plan, trace) = builder.new_scan().plan_with_trace().await.unwrap(); + let plan = builder.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); let mut planned_files = plan .splits() .iter() @@ -7129,8 +7130,8 @@ mod tests { "payload-1.blob".to_string(), ]; assert_eq!(planned_files, expected_planned_files); - assert_eq!(trace.manifest_entries_pruned_by_row_ranges, 4); - assert_eq!(trace.final_files, 3); + assert_eq!(scan_stats.manifest_entries_pruned_by_row_ranges, 4); + assert_eq!(scan_stats.final_files, 3); let batches = builder .new_read() diff --git a/crates/paimon/src/table/data_file_index_writer/tests.rs b/crates/paimon/src/table/data_file_index_writer/tests.rs index 8bc6cbdd3..f4d275748 100644 --- a/crates/paimon/src/table/data_file_index_writer/tests.rs +++ b/crates/paimon/src/table/data_file_index_writer/tests.rs @@ -1098,8 +1098,11 @@ async fn test_file_index_prunes_without_opening_data_file() { .unwrap(); let mut read_builder = table.new_read_builder(); read_builder.with_filter(predicate.clone()); - let (_, trace) = read_builder.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.final_files, 1, "statistics must retain the file"); + let scan_stats = { + let statistics_plan = read_builder.new_scan().plan().await.unwrap(); + statistics_plan.stats().clone() + }; + assert_eq!(scan_stats.final_files, 1, "statistics must retain the file"); storage.data_accesses.store(0, Ordering::SeqCst); assert!(query(&table, true, Some(predicate.clone())) .await @@ -1143,8 +1146,11 @@ async fn test_pk_file_index_prunes_before_sort_merge_opens_data() { .unwrap(); let mut read_builder = table.new_read_builder(); read_builder.with_filter(predicate.clone()); - let (_, trace) = read_builder.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.final_files, 1, "stats must retain the file"); + let scan_stats = { + let statistics_plan = read_builder.new_scan().plan().await.unwrap(); + statistics_plan.stats().clone() + }; + assert_eq!(scan_stats.final_files, 1, "stats must retain the file"); storage.data_accesses.store(0, Ordering::SeqCst); assert!(pk_query(&table, true, Some(predicate.clone())) .await @@ -1186,8 +1192,11 @@ async fn test_data_evolution_file_index_prunes_independent_file() { .unwrap(); let mut read_builder = table.new_read_builder(); read_builder.with_filter(predicate.clone()); - let (_, trace) = read_builder.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.final_files, 1, "stats must retain the file"); + let scan_stats = { + let statistics_plan = read_builder.new_scan().plan().await.unwrap(); + statistics_plan.stats().clone() + }; + assert_eq!(scan_stats.final_files, 1, "stats must retain the file"); storage.data_accesses.store(0, Ordering::SeqCst); assert!(query(&table, true, Some(predicate.clone())) .await @@ -1586,8 +1595,11 @@ async fn test_range_bitmap_append_range_pruning() { .unwrap(); let mut scan = table.new_read_builder(); scan.with_filter(missing.clone()); - let (_, trace) = scan.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.final_files, 1, "statistics must retain the file"); + let scan_stats = { + let statistics_plan = scan.new_scan().plan().await.unwrap(); + statistics_plan.stats().clone() + }; + assert_eq!(scan_stats.final_files, 1, "statistics must retain the file"); storage.data_accesses.store(0, Ordering::SeqCst); assert!(query(&table, true, Some(missing.clone())).await.is_empty()); assert_eq!(storage.data_accesses.load(Ordering::SeqCst), 0); diff --git a/crates/paimon/src/table/format_table_scan.rs b/crates/paimon/src/table/format_table_scan.rs index b5a1d1a0c..054b0df2e 100644 --- a/crates/paimon/src/table/format_table_scan.rs +++ b/crates/paimon/src/table/format_table_scan.rs @@ -24,7 +24,7 @@ use super::format_partition::{ FormatTablePartitionPaths, }; use super::format_partition_location::FormatPartitionLocations; -use super::{Plan, RESTEnv, ScanTrace, Table}; +use super::{Plan, RESTEnv, ScanStats, Table}; use crate::api::RestError; use crate::spec::stats::BinaryTableStats; use crate::spec::{ @@ -66,15 +66,9 @@ impl<'a> FormatTableScan<'a> { pub(crate) async fn plan(&self) -> crate::Result { self.ensure_query_auth_allowed()?; - self.plan_inner(None).await - } - - pub(crate) async fn plan_with_trace(&self) -> crate::Result<(Plan, ScanTrace)> { - self.ensure_query_auth_allowed()?; - let mut trace = ScanTrace::default(); - let plan = self.plan_inner(Some(&mut trace)).await?; - trace.planned_data_file_bytes = plan.planned_data_file_bytes(); - Ok((plan, trace)) + let mut stats = ScanStats::default(); + let plan = self.plan_inner(&mut stats).await?; + Ok(plan.with_scan_stats(stats)) } fn ensure_query_auth_allowed(&self) -> crate::Result<()> { @@ -82,7 +76,7 @@ impl<'a> FormatTableScan<'a> { CoreOptions::new(self.table.schema().options()).ensure_read_authorized() } - async fn plan_inner(&self, trace: Option<&mut ScanTrace>) -> crate::Result { + async fn plan_inner(&self, scan_stats: &mut ScanStats) -> crate::Result { if self.row_ranges.is_some() { return Err(crate::Error::Unsupported { message: "Row ranges are not supported for format tables".to_string(), @@ -155,9 +149,7 @@ impl<'a> FormatTableScan<'a> { }); splits = self.apply_limit_pushdown(splits); - if let Some(trace) = trace { - trace.record_final_plan(splits.len(), splits.len(), splits.len()); - } + scan_stats.record_split_planning(splits.len(), false); Ok(Plan::new(splits)) } diff --git a/crates/paimon/src/table/global_index_result.rs b/crates/paimon/src/table/global_index_result.rs index 914b09a31..f749f850e 100644 --- a/crates/paimon/src/table/global_index_result.rs +++ b/crates/paimon/src/table/global_index_result.rs @@ -373,33 +373,32 @@ impl GlobalIndexResult { let Some(scores) = self.score_map()? else { return Ok(plan); }; - let snapshot_id = plan.snapshot_id(); - let splits = plan - .into_splits() - .into_iter() - .map(|split| { - let selected = split - .row_ranges() - .ok_or_else(|| crate::Error::DataInvalid { - message: "Scored global-index splits must carry row ranges".into(), - source: None, - })?; - let mut values = Vec::new(); - for range in selected { - for id in range.from()..=range.to() { - values.push(*scores.get(&id).ok_or_else(|| crate::Error::DataInvalid { - message: format!("Missing score for selected global row ID {id}"), + plan.map_splits(|splits| { + splits + .into_iter() + .map(|split| { + let selected = split + .row_ranges() + .ok_or_else(|| crate::Error::DataInvalid { + message: "Scored global-index splits must carry row ranges".into(), source: None, - })?); + })?; + let mut values = Vec::new(); + for range in selected { + for id in range.from()..=range.to() { + values.push(*scores.get(&id).ok_or_else(|| { + crate::Error::DataInvalid { + message: format!( + "Missing score for selected global row ID {id}" + ), + source: None, + } + })?); + } } - } - split.with_scores(values) - }) - .collect::>>()?; - let plan = Plan::new(splits); - Ok(match snapshot_id { - Some(id) => plan.with_snapshot_id(id), - None => plan, + split.with_scores(values) + }) + .collect::>>() }) } } diff --git a/crates/paimon/src/table/global_index_result/tests.rs b/crates/paimon/src/table/global_index_result/tests.rs index 100ff6abc..815a84a32 100644 --- a/crates/paimon/src/table/global_index_result/tests.rs +++ b/crates/paimon/src/table/global_index_result/tests.rs @@ -48,8 +48,9 @@ async fn pk_supplied_result_preserves_snapshot_splits_and_ignores_planning_hints .with_shard(1, 2) .unwrap(); let plan = scan.plan().await.unwrap(); - let (traced, trace) = scan.plan_with_trace().await.unwrap(); - for plan in [plan, traced] { + let stats_plan = scan.clone().plan().await.unwrap(); + let scan_stats = stats_plan.stats().clone(); + for plan in [plan, stats_plan] { assert_eq!(plan.snapshot_id(), Some(42)); assert_eq!(plan.splits(), std::slice::from_ref(&split)); assert_eq!( @@ -57,12 +58,11 @@ async fn pk_supplied_result_preserves_snapshot_splits_and_ignores_planning_hints split.serialize_split_v1().unwrap() ); } - assert_eq!(trace.snapshot_id, Some(42)); - assert_eq!(trace.manifest_entries_read, 0); - assert_eq!(trace.final_splits, 1); - assert_eq!(trace.final_files, split.data_files().len()); + assert_eq!(scan_stats.manifest_entries_read, 0); + assert_eq!(scan_stats.final_splits, 1); + assert_eq!(scan_stats.final_files, split.data_files().len()); assert_eq!( - trace.planned_data_file_bytes, + scan_stats.planned_data_file_bytes, split .data_files() .iter() @@ -138,13 +138,19 @@ async fn de_scores_follow_final_ranges_after_distribution_and_limit() { GlobalIndexResult::from_scored_row_ids(ScoredRowIds::new(vec![2, 0], vec![0.8, 0.1])); let mut builder = table.new_read_builder(); builder.with_projection(&["id"]).unwrap(); - let plan = builder + let scan = builder .new_scan() .with_global_index_result(hits.clone()) - .unwrap() - .plan() - .await .unwrap(); + let plan = scan.plan().await.unwrap(); + let statistics_plan = scan.clone().plan().await.unwrap(); + assert_eq!(statistics_plan.splits(), plan.splits()); + let scan_stats = statistics_plan.stats(); + assert_eq!(statistics_plan.snapshot_id(), plan.snapshot_id()); + assert_eq!( + scan_stats.planned_data_file_bytes, + plan.planned_data_file_bytes() + ); let ranges = plan .splits() .iter() @@ -279,7 +285,7 @@ async fn supplied_pk_result_cannot_bypass_required_authorization() { )) .unwrap(); assert!(scan.plan().await.is_err()); - assert!(scan.plan_with_trace().await.is_err()); + assert!(scan.clone().plan().await.is_err()); } #[tokio::test] @@ -331,6 +337,14 @@ async fn de_combined_incremental_results_retain_the_window_and_scores() { .unwrap(); assert!(scan.plan().await.is_err()); let plan = scan.plan_combined().await.unwrap(); + let statistics_plan = scan.plan_combined().await.unwrap(); + assert_eq!(statistics_plan.splits(), plan.splits()); + let scan_stats = statistics_plan.stats(); + assert_eq!(statistics_plan.snapshot_id(), Some(end)); + assert_eq!( + scan_stats.planned_data_file_bytes, + plan.planned_data_file_bytes() + ); assert_eq!(plan.snapshot_id(), Some(end)); assert_eq!(plan.splits().len(), 1); assert_eq!( diff --git a/crates/paimon/src/table/incremental_scan.rs b/crates/paimon/src/table/incremental_scan.rs index b56a65a67..70ece9224 100644 --- a/crates/paimon/src/table/incremental_scan.rs +++ b/crates/paimon/src/table/incremental_scan.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use super::{DataSplit, GlobalIndexResult, Plan, ScanTrace, SnapshotManager, Table, TableScan}; +use super::{DataSplit, GlobalIndexResult, Plan, ScanStats, SnapshotManager, Table, TableScan}; use crate::spec::{CommitKind, CoreOptions}; /// Java IncrementalDeltaStartingScanner.betweenTimestamps. Resolve both bounds @@ -358,17 +358,22 @@ impl<'a> IncrementalScan<'a> { /// represented by an ordinary split list because each unit contains a /// before/after pair. pub async fn plan_combined(&self) -> crate::Result { - self.plan_combined_with_trace(None).await + let mut stats = ScanStats { + limit: self.scan.limit_hint(), + ..Default::default() + }; + let plan = self.plan_combined_inner(&mut stats).await?; + Ok(plan.with_scan_stats(stats)) } - pub(crate) async fn plan_combined_with_trace( + pub(crate) async fn plan_combined_inner( &self, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { let mode = self.resolve_mode(); match mode { IncrementalScanMode::Delta | IncrementalScanMode::Changelog => { - self.plan_combined_events(mode, trace).await + self.plan_combined_events(mode, scan_stats).await } IncrementalScanMode::Diff => Err(crate::Error::Unsupported { message: "Combined incremental planning does not support Diff mode; Diff requires before/after split pairs" @@ -393,13 +398,13 @@ impl<'a> IncrementalScan<'a> { message: "Combined incremental planning only supports Delta mode".to_string(), }); } - self.plan_combined_events(mode, None).await + self.plan_combined().await } async fn plan_combined_events( &self, mode: IncrementalScanMode, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { let scan = self.scan.authorize_snapshot_read().await?; // Row positions and shuffled chunks refer to one APPEND-delta row @@ -413,9 +418,7 @@ impl<'a> IncrementalScan<'a> { } self.validate_snapshot_range(mode).await?; if mode == IncrementalScanMode::Changelog && self.start_exclusive == self.end_inclusive { - if let Some(trace) = trace { - trace.record_final_plan(0, 0, 0); - } + scan_stats.record_split_planning(0, false); // Changelog's empty range may end at earliest-1, which is a // valid boundary even though that snapshot need not exist. return Ok(Plan::new(Vec::new()).with_snapshot_id(self.end_inclusive)); @@ -452,7 +455,7 @@ impl<'a> IncrementalScan<'a> { .map(str::to_owned) .collect(); let plan = scan - .plan_incremental_lists_with_trace(&end_snapshot, manifest_lists, trace) + .plan_incremental_lists(&end_snapshot, manifest_lists, scan_stats) .await?; self.scan.attach_global_index_scores(plan) } @@ -534,10 +537,7 @@ impl<'a> IncrementalScan<'a> { if snapshot.changelog_manifest_list().is_none() { continue; } - let plan = self - .scan - .plan_snapshot_changelog_with_trace(&snapshot, None) - .await?; + let plan = self.scan.plan_snapshot_changelog(&snapshot).await?; splits.extend(plan.splits().iter().cloned().map(IncrementalSplit::Data)); } IncrementalPlan::try_new(mode, splits) diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index abc961122..08a847430 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -140,7 +140,7 @@ pub(crate) mod row_id_predicate; mod row_kind_generator; mod row_position_selection; mod row_sidecar; -mod scan_trace; +mod scan_stats; pub(crate) mod schema_manager; mod search_scan_split; mod shard_table_updater; @@ -227,7 +227,7 @@ pub use postpone_fixed_bucket_write::{ pub use postpone_fixed_bucket_write_builder::PostponeFixedBucketWriteBuilder; pub use read_builder::ReadBuilder; pub use rest_env::RESTEnv; -pub use scan_trace::ScanTrace; +pub use scan_stats::ScanStats; pub use schema_manager::SchemaManager; pub use search_scan_split::SearchScanSplit; pub use shard_table_updater::ShardTableUpdater; diff --git a/crates/paimon/src/table/mosaic_table_write_tests.rs b/crates/paimon/src/table/mosaic_table_write_tests.rs index e26a34711..6770ccb60 100644 --- a/crates/paimon/src/table/mosaic_table_write_tests.rs +++ b/crates/paimon/src/table/mosaic_table_write_tests.rs @@ -380,9 +380,10 @@ async fn append_mosaic_stats_prune_manifest_files() { .unwrap(); let mut read = table.new_read_builder(); read.with_filter(predicate); - let (plan, trace) = read.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.final_files, 1, "scan trace: {trace:?}"); - assert!(trace.manifest_entries_pruned_by_data_stats >= 1); + let plan = read.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); + assert_eq!(scan_stats.final_files, 1, "scan scan_stats: {scan_stats:?}"); + assert!(scan_stats.manifest_entries_pruned_by_data_stats >= 1); let batches: Vec = read .new_read() .unwrap() @@ -709,9 +710,10 @@ async fn append_mosaic_without_stats_keeps_files_in_filtered_plan() { .unwrap(); let mut reader = table.new_read_builder(); reader.with_filter(predicate); - let (_plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.final_files, 2, "scan trace: {trace:?}"); - assert_eq!(trace.manifest_entries_pruned_by_data_stats, 0); + let _plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = _plan.stats().clone(); + assert_eq!(scan_stats.final_files, 2, "scan scan_stats: {scan_stats:?}"); + assert_eq!(scan_stats.manifest_entries_pruned_by_data_stats, 0); } #[tokio::test] @@ -751,8 +753,9 @@ async fn append_mosaic_all_null_statistics_remain_conservative() { .unwrap(); let mut reader = table.new_read_builder(); reader.with_filter(predicate); - let (plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.final_files, 1); + let plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); + assert_eq!(scan_stats.final_files, 1); let batches: Vec = reader .new_read() .unwrap() @@ -887,8 +890,9 @@ async fn append_mosaic_filters_rows_when_predicate_column_has_no_stats() { .unwrap(); let mut reader = table.new_read_builder(); reader.with_filter(predicate); - let (plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.final_files, 1); + let plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); + assert_eq!(scan_stats.final_files, 1); let batches: Vec = reader .new_read() .unwrap() diff --git a/crates/paimon/src/table/pk_full_text_scan.rs b/crates/paimon/src/table/pk_full_text_scan.rs index 679d20c25..165d981db 100644 --- a/crates/paimon/src/table/pk_full_text_scan.rs +++ b/crates/paimon/src/table/pk_full_text_scan.rs @@ -288,22 +288,18 @@ impl<'a> PrimaryKeyFullTextScan<'a> { read_builder.with_filter(filter.clone()); } // Plan the data splits and capture the snapshot the scan pinned in one - // pass. The trace carries the resolved snapshot id even when the scan + // pass. The plan carries the resolved snapshot id even when the scan // yields zero data splits, so the plan reports its real snapshot id // (required by the cross-route snapshot-consistency guard) instead of // deriving it from a first split that may not exist. - let (data_plan, trace) = read_builder - .new_scan() - .with_scan_all_files() - .plan_with_trace() - .await?; + let data_plan = read_builder.new_scan().with_scan_all_files().plan().await?; let data_splits = data_plan.splits().to_vec(); // No snapshot at all (table never written): nothing to search and no // snapshot to pin. The empty split list makes every downstream consumer // treat this as "no candidates", so this is the only plan without a real // snapshot id. - let Some(snapshot_id) = trace.snapshot_id else { + let Some(snapshot_id) = data_plan.snapshot_id() else { return Ok(PrimaryKeyFullTextScanPlan { snapshot_id: 0, splits: Vec::new(), diff --git a/crates/paimon/src/table/read_builder.rs b/crates/paimon/src/table/read_builder.rs index 756764c34..76a0b5388 100644 --- a/crates/paimon/src/table/read_builder.rs +++ b/crates/paimon/src/table/read_builder.rs @@ -75,6 +75,9 @@ pub(super) fn split_scan_predicates( fn bucket_predicate(table: &Table, filter: &Predicate) -> Option { let core_options = CoreOptions::new(table.schema().options()); + if core_options.bucket() < 1 && core_options.bucket() != -2 { + return None; + } let bucket_keys = core_options.bucket_key().unwrap_or_else(|| { if table.schema().trimmed_primary_keys().is_empty() { Vec::new() @@ -86,18 +89,22 @@ fn bucket_predicate(table: &Table, filter: &Predicate) -> Option { return None; } + let mut referenced = std::collections::HashSet::new(); + filter.collect_leaf_field_indices(&mut referenced); let has_all_bucket_fields = bucket_keys.iter().all(|key| { table .schema() .fields() .iter() - .any(|field| field.name() == key) + .enumerate() + .any(|(index, field)| field.name() == key && referenced.contains(&index)) }); if !has_all_bucket_fields { return None; } - extract_predicate_for_keys(filter, table.schema().fields(), &bucket_keys) + // Preserve partition leaves until the entry's concrete partition is known. + Some(filter.clone()) } fn normalize_filter(table: &Table, filter: Predicate) -> NormalizedFilter { diff --git a/crates/paimon/src/table/scan_trace.rs b/crates/paimon/src/table/scan_stats.rs similarity index 73% rename from crates/paimon/src/table/scan_trace.rs rename to crates/paimon/src/table/scan_stats.rs index 450638004..dc59d3155 100644 --- a/crates/paimon/src/table/scan_trace.rs +++ b/crates/paimon/src/table/scan_stats.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Scan planning trace counters. +//! Scan planning statistics. use std::fmt; @@ -23,11 +23,13 @@ use std::fmt; /// /// The counters are intended for explain output and regression tests. They /// describe pruning at metadata planning time only; reader-side Parquet row -/// group pruning and DataFusion residual filters are outside this trace. +/// group pruning and DataFusion residual filters are outside these statistics. +/// Every scan stores this type on [`super::Plan::stats`]. Counters describe +/// normal planning work, retaining manifest pruning and LIMIT stopping. +/// Snapshot metadata is available separately from [`super::Plan::snapshot_id`]. #[non_exhaustive] #[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ScanTrace { - pub snapshot_id: Option, +pub struct ScanStats { pub base_manifest_files: usize, pub delta_manifest_files: usize, pub manifest_files_before_partition_pruning: usize, @@ -47,8 +49,11 @@ pub struct ScanTrace { pub data_evolution_groups_before_stats: usize, pub data_evolution_groups_pruned_by_stats: usize, pub data_evolution_groups_pruned_by_row_ranges: usize, + /// Candidates actually constructed; excludes work skipped by LIMIT stopping. pub split_candidates_built: usize, + /// Whether a LIMIT hint stopped planning before all candidates were built. pub limit_early_stopped: bool, + /// Candidates encountered before applying the LIMIT hint, not a full-scan total. pub splits_before_limit: usize, pub splits_after_limit: usize, pub final_splits: usize, @@ -58,51 +63,25 @@ pub struct ScanTrace { pub limit: Option, } -impl ScanTrace { +impl ScanStats { pub(crate) fn record_manifest_lists(&mut self, base_count: usize, delta_count: usize) { self.base_manifest_files = base_count; self.delta_manifest_files = delta_count; self.manifest_files_before_partition_pruning = base_count + delta_count; } - pub(crate) fn record_final_plan( - &mut self, - splits_before_limit: usize, - splits: usize, - files: usize, - ) { - self.record_final_plan_with_limit( - splits_before_limit, - splits_before_limit, - splits, - files, - false, - ); - } - - pub(crate) fn record_final_plan_with_limit( - &mut self, - split_candidates_built: usize, - splits_before_limit: usize, - splits: usize, - files: usize, - limit_early_stopped: bool, - ) { - self.split_candidates_built = split_candidates_built; + pub(crate) fn record_split_planning(&mut self, candidates: usize, limit_early_stopped: bool) { + self.split_candidates_built = candidates; + self.splits_before_limit = candidates; self.limit_early_stopped = limit_early_stopped; - self.splits_before_limit = splits_before_limit; - self.splits_after_limit = splits; - self.final_splits = splits; - self.final_files = files; } } -impl fmt::Display for ScanTrace { +impl fmt::Display for ScanStats { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "snapshot={:?}, manifests={}/{}, manifest_row_range_pruned={}, entries_read={}, bucket_pruned={}, partition_pruned={}, entry_row_range_pruned={}, data_stats_pruned={}, cross_schema_pruned={}, split_candidates_built={}, limit_early_stopped={}, splits_before_limit={}, splits_after_limit={}, files={}, data_file_bytes={}", - self.snapshot_id, + "manifests={}/{}, manifest_row_range_pruned={}, entries_read={}, bucket_pruned={}, partition_pruned={}, entry_row_range_pruned={}, data_stats_pruned={}, cross_schema_pruned={}, split_candidates_built={}, limit_early_stopped={}, splits_before_limit={}, splits_after_limit={}, files={}, data_file_bytes={}", self.manifest_files_after_partition_pruning, self.manifest_files_before_partition_pruning, self.manifest_files_pruned_by_row_ranges, diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/tests.rs b/crates/paimon/src/table/sorted_global_index_build_builder/tests.rs index 74ac730d6..2b463aded 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/tests.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder/tests.rs @@ -1395,7 +1395,8 @@ async fn test_global_index_prunes_during_manifest_read() { .unwrap(); let mut read_builder = table.new_read_builder(); read_builder.with_filter(predicate); - let (plan, trace) = read_builder.new_scan().plan_with_trace().await.unwrap(); + let plan = read_builder.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); assert_eq!( plan.splits() @@ -1405,13 +1406,13 @@ async fn test_global_index_prunes_during_manifest_read() { 1 ); assert_eq!( - trace.manifest_files_pruned_by_row_ranges, + scan_stats.manifest_files_pruned_by_row_ranges, expected_manifest_pruned ); - assert_eq!(trace.manifest_entries_read, expected_entries_read); - assert_eq!(trace.manifest_entries_pruned_by_row_ranges, 0); + assert_eq!(scan_stats.manifest_entries_read, expected_entries_read); + assert_eq!(scan_stats.manifest_entries_pruned_by_row_ranges, 0); assert_eq!( - trace.manifest_entries_after_manifest_filters, + scan_stats.manifest_entries_after_manifest_filters, expected_entries_read ); } @@ -1543,17 +1544,18 @@ async fn test_empty_global_index_ranges_skip_legacy_manifests() { let plan = read_builder.new_scan().plan().await.unwrap(); assert!(plan.splits().is_empty()); - let (traced_plan, trace) = read_builder.new_scan().plan_with_trace().await.unwrap(); + let stats_plan = read_builder.new_scan().plan().await.unwrap(); + let scan_stats = stats_plan.stats().clone(); let delta_error = read_builder .new_scan() .plan_snapshot_delta(&snapshot) .await .unwrap_err(); - assert!(traced_plan.splits().is_empty()); - assert_eq!(trace.manifest_entries_read, 0); - assert_eq!(trace.final_splits, 0); - assert_eq!(trace.final_files, 0); + assert!(stats_plan.splits().is_empty()); + assert_eq!(scan_stats.manifest_entries_read, 0); + assert_eq!(scan_stats.final_splits, 0); + assert_eq!(scan_stats.final_files, 0); // A snapshot index may short-circuit a batch read, but cannot hide // an invalid historical event file from incremental planning. assert!( @@ -1602,7 +1604,8 @@ async fn test_detail_mode_defers_manifest_pruning_for_unindexed_ranges() { .unwrap(); let mut read_builder = table.new_read_builder(); read_builder.with_filter(predicate); - let (plan, trace) = read_builder.new_scan().plan_with_trace().await.unwrap(); + let plan = read_builder.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); let planned_ranges = merge_row_ranges( plan.splits() .iter() @@ -1622,8 +1625,8 @@ async fn test_detail_mode_defers_manifest_pruning_for_unindexed_ranges() { planned_ranges, vec![RowRange::new(0, 0), RowRange::new(2, 3)] ); - assert_eq!(trace.manifest_files_pruned_by_row_ranges, 0); - assert_eq!(trace.manifest_entries_read, 2); + assert_eq!(scan_stats.manifest_files_pruned_by_row_ranges, 0); + assert_eq!(scan_stats.manifest_entries_read, 2); } #[tokio::test] diff --git a/crates/paimon/src/table/source.rs b/crates/paimon/src/table/source.rs index 355b76949..25bf432df 100644 --- a/crates/paimon/src/table/source.rs +++ b/crates/paimon/src/table/source.rs @@ -1601,14 +1601,20 @@ impl Default for DataSplitBuilder { pub struct Plan { splits: Vec, snapshot_id: Option, + scan_stats: super::ScanStats, } impl Plan { pub fn new(splits: Vec) -> Self { - Self { + let mut plan = Self { splits, snapshot_id: None, - } + scan_stats: super::ScanStats::default(), + }; + plan.refresh_final_stats(); + plan.scan_stats.split_candidates_built = plan.splits.len(); + plan.scan_stats.splits_before_limit = plan.splits.len(); + plan } pub(crate) fn with_snapshot_id(mut self, snapshot_id: i64) -> Self { @@ -1627,6 +1633,46 @@ impl Plan { &self.splits } + /// Lightweight statistics collected in the same planning pass. + pub fn stats(&self) -> &super::ScanStats { + &self.scan_stats + } + + pub(crate) fn with_scan_stats(mut self, mut stats: super::ScanStats) -> Self { + stats.final_splits = self.scan_stats.final_splits; + stats.splits_after_limit = self.scan_stats.splits_after_limit; + stats.final_files = self.scan_stats.final_files; + stats.planned_data_file_bytes = self.scan_stats.planned_data_file_bytes; + self.scan_stats = stats; + self + } + + fn refresh_final_stats(&mut self) { + let (files, bytes) = self.splits.iter().flat_map(DataSplit::data_files).fold( + (0, 0_u64), + |(files, bytes), file| { + ( + files + 1, + bytes.saturating_add(u64::try_from(file.file_size).unwrap_or(0)), + ) + }, + ); + self.scan_stats.final_splits = self.splits.len(); + self.scan_stats.splits_after_limit = self.splits.len(); + self.scan_stats.final_files = files; + self.scan_stats.planned_data_file_bytes = bytes; + } + + /// Transform splits without dropping the snapshot or collected statistics. + pub(crate) fn map_splits( + mut self, + transform: impl FnOnce(Vec) -> crate::Result>, + ) -> crate::Result { + self.splits = transform(self.splits)?; + self.refresh_final_stats(); + Ok(self) + } + /// Stamps the grant every split of this plan was authorized under. pub(crate) fn planned(mut self, grant: Option>) -> Self { if grant.is_some() { @@ -1645,11 +1691,7 @@ impl Plan { /// file size. Totals larger than [`u64::MAX`] saturate at that value so /// callers cannot under-count an oversized plan due to integer overflow. pub fn planned_data_file_bytes(&self) -> u64 { - self.splits - .iter() - .flat_map(DataSplit::data_files) - .filter_map(|file| u64::try_from(file.file_size).ok()) - .fold(0, u64::saturating_add) + self.scan_stats.planned_data_file_bytes } /// Consume this plan and return its splits without cloning their file metadata. @@ -1762,6 +1804,54 @@ mod tests { let plan = Plan::new(vec![split(vec![first, second, overflow, unknown], true)]); assert_eq!(plan.planned_data_file_bytes(), u64::MAX); + assert_eq!(plan.stats().final_files, 4); + assert_eq!(plan.stats().planned_data_file_bytes, u64::MAX); + } + + #[test] + fn transformed_plan_refreshes_totals_and_preserves_planning_work() { + let plan = Plan::new(vec![ + split( + vec![ + file("a.parquet", 10, Some(0)), + file("b.parquet", 10, Some(10)), + ], + true, + ), + split(vec![file("c.parquet", 10, Some(20))], true), + ]) + .with_snapshot_id(7) + .with_scan_stats(super::super::ScanStats { + manifest_entries_read: 20, + split_candidates_built: 2, + splits_before_limit: 2, + limit_early_stopped: true, + limit: Some(10), + ..Default::default() + }); + assert_eq!(plan.stats().final_files, 3); + assert_eq!(plan.planned_data_file_bytes(), 384); + + let selected = plan + .map_splits(|mut splits| Ok(vec![splits.pop().unwrap()])) + .unwrap(); + assert_eq!(selected.snapshot_id(), Some(7)); + assert_eq!(selected.stats().manifest_entries_read, 20); + assert_eq!(selected.stats().split_candidates_built, 2); + assert_eq!(selected.stats().splits_before_limit, 2); + assert!(selected.stats().limit_early_stopped); + assert_eq!(selected.stats().limit, Some(10)); + assert_eq!(selected.stats().final_splits, 1); + assert_eq!(selected.stats().splits_after_limit, 1); + assert_eq!(selected.stats().final_files, 1); + assert_eq!(selected.planned_data_file_bytes(), 128); + + let empty = selected.map_splits(|_| Ok(Vec::new())).unwrap(); + assert_eq!(empty.snapshot_id(), Some(7)); + assert_eq!(empty.stats().manifest_entries_read, 20); + assert_eq!(empty.stats().final_splits, 0); + assert_eq!(empty.stats().final_files, 0); + assert_eq!(empty.planned_data_file_bytes(), 0); } #[test] diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index c949080a1..4e89f05d9 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -20,7 +20,7 @@ //! Reference: [pypaimon.read.table_scan.TableScan](https://github.com/apache/paimon/blob/release-1.3/paimon-python/pypaimon/read/table_scan.py) //! and [FullStartingScanner](https://github.com/apache/paimon/blob/release-1.3/paimon-python/pypaimon/read/scanner/full_starting_scanner.py). -use super::bucket_filter::compute_target_buckets; +use super::bucket_filter::{compute_target_buckets, BucketSelector}; use super::chunk_shuffle::{chunk_shuffle_splits, ChunkShuffle}; use super::format_table_scan::FormatTableScan; use super::global_index_result::GlobalIndexResult; @@ -52,7 +52,7 @@ use crate::table::source::{ any_range_overlaps_file, intersect_ranges_with_file, merge_row_ranges, DataSplit, DataSplitBuilder, DeletionFile, PartitionBucket, Plan, RowRange, }; -use crate::table::ScanTrace; +use crate::table::ScanStats; use futures::{StreamExt, TryStreamExt}; use indexmap::IndexMap; use std::borrow::Cow; @@ -161,6 +161,9 @@ async fn read_manifest_bytes_with_sidecar( }; let mut bucket_cache: HashMap>> = HashMap::new(); let mut bucket_block_filter = |bucket: i32, total_buckets: i32| { + if bucket < 0 || total_buckets <= 0 { + return true; + } let Some(predicate) = bucket_predicate else { return true; }; @@ -226,8 +229,7 @@ enum ManifestListSource<'a> { /// - Data-level stats pruning per entry (current schema only, cross-schema fail-open) #[allow(clippy::too_many_arguments)] async fn read_all_manifest_entries( - file_io: &FileIO, - table_path: &str, + table: &Table, source: ManifestListSource<'_>, skip_level_zero: bool, scan_all_files: bool, @@ -239,14 +241,18 @@ async fn read_all_manifest_entries( current_schema_id: i64, schema_fields: &[DataField], key_fields: &[String], - bucket_predicate: Option<&Predicate>, - bucket_key_fields: &[DataField], - bucket_function_type: BucketFunctionType, + bucket_selector: Option<&BucketSelector>, row_range_index: Option<&RowRangeIndex>, manifest_sidecar_enabled: bool, manifest_parallelism: usize, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result> { + let file_io = table.file_io(); + let table_path = table.location(); + let bucket_predicate = bucket_selector.and_then(BucketSelector::manifest_predicate); + let bucket_key_fields = bucket_selector.map_or(&[][..], BucketSelector::bucket_fields); + let bucket_function_type = + bucket_selector.map_or(BucketFunctionType::Default, BucketSelector::function); let incremental = !matches!(&source, ManifestListSource::Snapshot(_)); let (mut manifest_files, delta) = match source { ManifestListSource::Snapshot(snapshot) => futures::try_join!( @@ -269,10 +275,7 @@ async fn read_all_manifest_entries( (Vec::new(), delta) } }; - let mut trace = trace; - if let Some(trace) = trace.as_deref_mut() { - trace.record_manifest_lists(manifest_files.len(), delta.len()); - } + scan_stats.record_manifest_lists(manifest_files.len(), delta.len()); manifest_files.extend(delta); // Manifest-file-level partition stats pruning: skip entire manifest files @@ -281,10 +284,8 @@ async fn read_all_manifest_entries( if let Some(pf) = partition_filter { manifest_files.retain(|meta| pf.matches_manifest(meta, partition_fields)); } - if let Some(trace) = trace.as_deref_mut() { - trace.manifest_files_before_partition_pruning = manifest_files_before_partition_pruning; - trace.manifest_files_after_partition_pruning = manifest_files.len(); - } + scan_stats.manifest_files_before_partition_pruning = manifest_files_before_partition_pruning; + scan_stats.manifest_files_after_partition_pruning = manifest_files.len(); retain_manifest_buckets( &mut manifest_files, @@ -297,9 +298,7 @@ async fn read_all_manifest_entries( if let Some(index) = row_range_index { let before = manifest_files.len(); retain_manifest_row_ranges(&mut manifest_files, index); - if let Some(trace) = trace.as_deref_mut() { - trace.manifest_files_pruned_by_row_ranges = before - manifest_files.len(); - } + scan_stats.manifest_files_pruned_by_row_ranges = before - manifest_files.len(); } let manifest_path_prefix = format!("{}/{}", table_path.trim_end_matches('/'), MANIFEST_DIR); @@ -328,8 +327,7 @@ async fn read_all_manifest_entries( ) .await?; - // Per-task bucket cache (few distinct total_buckets values per manifest). - let mut bucket_cache: HashMap>> = HashMap::new(); + let mut bucket_selector = bucket_selector.cloned(); let mut counters = ManifestReadCounters::default(); let entries = crate::spec::avro::from_manifest_bytes_filtered_shared( @@ -342,33 +340,19 @@ async fn read_all_manifest_entries( counters.pruned_by_bucket += 1; return false; } - if let Some(pred) = bucket_predicate { - let targets = bucket_cache.entry(total_buckets).or_insert_with(|| { - compute_target_buckets( - pred, - bucket_key_fields, - bucket_function_type, - total_buckets, - ) - }); - if let Some(targets) = targets { - if !targets.contains(&bucket) { - counters.pruned_by_bucket += 1; - return false; - } - } + if bucket_selector.as_mut().is_some_and(|selector| { + !selector.test(partition_bytes, bucket, total_buckets) + }) { + counters.pruned_by_bucket += 1; + return false; } // Partition filter - if let Some(pf) = partition_filter { - match pf.matches_entry(partition_bytes) { - Ok(false) => { - counters.pruned_by_partition += 1; - return false; - } - Ok(true) => {} - Err(_) => {} - } + if partition_filter.is_some_and(|filter| { + !filter.matches_entry(partition_bytes).unwrap_or(true) + }) { + counters.pruned_by_partition += 1; + return false; } true @@ -427,17 +411,15 @@ async fn read_all_manifest_entries( all_entries = retain_manifest_entry_row_ranges(all_entries, index); counters.pruned_by_row_ranges = before - all_entries.len(); } - if let Some(trace) = trace { - trace.manifest_entries_read = counters.entries_read; - trace.manifest_entries_pruned_by_bucket = counters.pruned_by_bucket; - trace.manifest_entries_pruned_by_partition = counters.pruned_by_partition; - trace.manifest_entries_after_entry_pruning = counters.after_entry_pruning; - trace.manifest_entries_pruned_by_level = counters.pruned_by_level; - trace.manifest_entries_pruned_by_row_ranges = counters.pruned_by_row_ranges; - trace.manifest_entries_pruned_by_data_stats = counters.pruned_by_data_stats; - trace.manifest_entries_after_manifest_filters = counters.after_manifest_filters; - trace.manifest_entries_after_merge = manifest_entries_after_merge; - } + scan_stats.manifest_entries_read = counters.entries_read; + scan_stats.manifest_entries_pruned_by_bucket = counters.pruned_by_bucket; + scan_stats.manifest_entries_pruned_by_partition = counters.pruned_by_partition; + scan_stats.manifest_entries_after_entry_pruning = counters.after_entry_pruning; + scan_stats.manifest_entries_pruned_by_level = counters.pruned_by_level; + scan_stats.manifest_entries_pruned_by_row_ranges = counters.pruned_by_row_ranges; + scan_stats.manifest_entries_pruned_by_data_stats = counters.pruned_by_data_stats; + scan_stats.manifest_entries_after_manifest_filters = counters.after_manifest_filters; + scan_stats.manifest_entries_after_merge = manifest_entries_after_merge; Ok(all_entries) } @@ -1214,15 +1196,19 @@ impl<'a> TableScan<'a> { scan.split_selection .get_or_insert_with(Default::default) .snapshot_mode = Some(mode); + let mut stats = ScanStats::default(); let plan = match mode { SnapshotScanMode::All => { let fields = scan.projected_read_field_ids()?; - scan.plan_snapshot(snapshot, fields.as_ref(), None).await + scan.plan_snapshot(snapshot, fields.as_ref(), &mut stats) + .await + } + SnapshotScanMode::Delta => scan.plan_snapshot_delta(&snapshot, &mut stats).await, + SnapshotScanMode::Changelog => { + scan.plan_snapshot_changelog(&snapshot, &mut stats).await } - SnapshotScanMode::Delta => scan.plan_snapshot_delta(&snapshot).await, - SnapshotScanMode::Changelog => scan.plan_snapshot_changelog(&snapshot, None).await, }?; - scan.finish_plan(plan).await + scan.finish_plan(plan.with_scan_stats(stats)).await } pub(crate) fn new( @@ -1499,23 +1485,23 @@ impl<'a> TableScan<'a> { } } - pub async fn plan_with_trace(&self) -> crate::Result<(Plan, ScanTrace)> { + pub(super) fn limit_hint(&self) -> Option { match &self.0 { - TableScanKind::Paimon(scan) => Box::pin(scan.plan_with_trace()).await, - TableScanKind::Format(scan) => scan.plan_with_trace().await, + TableScanKind::Paimon(scan) => scan.limit, + TableScanKind::Format(_) => None, } } - pub(crate) async fn plan_incremental_lists_with_trace( + pub(crate) async fn plan_incremental_lists( &self, end_snapshot: &Snapshot, manifest_lists: Vec, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { match &self.0 { TableScanKind::Paimon(scan) => { let plan = scan - .plan_incremental_lists(end_snapshot, manifest_lists, trace) + .plan_incremental_lists(end_snapshot, manifest_lists, scan_stats) .await?; scan.finish_plan(plan).await } @@ -1540,7 +1526,11 @@ impl<'a> TableScan<'a> { /// Plan data splits from a snapshot's delta manifest list only. pub(crate) async fn plan_snapshot_delta(&self, snapshot: &Snapshot) -> crate::Result { match &self.0 { - TableScanKind::Paimon(scan) => scan.plan_snapshot_delta(snapshot).await, + TableScanKind::Paimon(scan) => { + let mut stats = ScanStats::default(); + let plan = scan.plan_snapshot_delta(snapshot, &mut stats).await?; + Ok(plan.with_scan_stats(stats)) + } TableScanKind::Format(_) => Err(crate::Error::Unsupported { message: "Format tables do not support incremental delta scan".to_string(), }), @@ -1548,13 +1538,13 @@ impl<'a> TableScan<'a> { } /// Plan data splits from a snapshot's changelog manifest list only. - pub(crate) async fn plan_snapshot_changelog_with_trace( - &self, - snapshot: &Snapshot, - trace: Option<&mut ScanTrace>, - ) -> crate::Result { + pub(crate) async fn plan_snapshot_changelog(&self, snapshot: &Snapshot) -> crate::Result { match &self.0 { - TableScanKind::Paimon(scan) => scan.plan_snapshot_changelog(snapshot, trace).await, + TableScanKind::Paimon(scan) => { + let mut stats = ScanStats::default(); + let plan = scan.plan_snapshot_changelog(snapshot, &mut stats).await?; + Ok(plan.with_scan_stats(stats)) + } TableScanKind::Format(_) => Err(crate::Error::Unsupported { message: "Format tables do not support incremental changelog scan".to_string(), }), @@ -1632,6 +1622,7 @@ struct PaimonTableScan<'a> { /// Optional limit on the number of rows to return. /// When set, the scan will try to return only enough splits to satisfy the limit. limit: Option, + /// Statistics collection is requested explicitly. row_ranges: Option>, /// Mutually exclusive row-position or fixed-row chunk transformation. /// Boxed because scans normally use neither, and keeping the cold payload @@ -1832,8 +1823,10 @@ impl<'a> PaimonTableScan<'a> { /// for `scan.version`; the strict selectors mirror Java's typed /// `scan.snapshot-id` / `scan.tag-name` handling. pub async fn plan(&self) -> crate::Result { - let plan = self.plan_selection().await?; - self.attach_index_scores(plan) + let mut stats = ScanStats::default(); + let plan = self.plan_selection(&mut stats).await?; + let plan = self.attach_index_scores(plan)?; + Ok(plan.with_scan_stats(stats)) } fn attach_index_scores(&self, plan: Plan) -> crate::Result { @@ -1845,9 +1838,11 @@ impl<'a> PaimonTableScan<'a> { } } - async fn plan_selection(&self) -> crate::Result { + async fn plan_selection(&self, stats: &mut ScanStats) -> crate::Result { let scan = self.authorized_scan().await?; + stats.limit = scan.limit; if let Some(plan) = scan.supplied_split_plan() { + stats.record_split_planning(plan.splits().len(), false); return Ok(plan); } if let Some((start, end)) = scan @@ -1856,13 +1851,12 @@ impl<'a> PaimonTableScan<'a> { .core_options() .incremental_timestamp_window()? { - let plan = scan.plan_timestamp_window(start, end, None).await?; - return Ok(plan); + return scan.plan_timestamp_window(start, end, stats).await; } let data_evolution_read_field_ids = scan.projected_read_field_ids()?; let plan = match super::time_travel::resolve_snapshot(&scan.table).await? { Some(snapshot) => { - scan.plan_snapshot(snapshot, data_evolution_read_field_ids.as_ref(), None) + scan.plan_snapshot(snapshot, data_evolution_read_field_ids.as_ref(), stats) .await? } None => Plan::new(Vec::new()), @@ -1870,60 +1864,6 @@ impl<'a> PaimonTableScan<'a> { scan.finish_plan(plan).await } - /// Plan the full scan and return metadata-pruning trace counters. - pub async fn plan_with_trace(&self) -> crate::Result<(Plan, ScanTrace)> { - let (plan, trace) = self.plan_selection_with_trace().await?; - Ok((self.attach_index_scores(plan)?, trace)) - } - - async fn plan_selection_with_trace(&self) -> crate::Result<(Plan, ScanTrace)> { - let scan = self.authorized_scan().await?; - let mut trace = ScanTrace { - limit: scan.limit, - ..Default::default() - }; - if let Some(plan) = scan.supplied_split_plan() { - trace.snapshot_id = plan.snapshot_id(); - trace.planned_data_file_bytes = plan.planned_data_file_bytes(); - trace.record_final_plan( - plan.splits().len(), - plan.splits().len(), - plan.splits().iter().map(|s| s.data_files().len()).sum(), - ); - return Ok((plan, trace)); - } - if let Some((start, end)) = scan - .table - .schema() - .core_options() - .incremental_timestamp_window()? - { - let plan = scan - .plan_timestamp_window(start, end, Some(&mut trace)) - .await?; - trace.snapshot_id = plan.snapshot_id(); - trace.planned_data_file_bytes = plan.planned_data_file_bytes(); - return Ok((plan, trace)); - } - let data_evolution_read_field_ids = scan.projected_read_field_ids()?; - let plan = match super::time_travel::resolve_snapshot(&scan.table).await? { - Some(snapshot) => { - trace.snapshot_id = Some(snapshot.id()); - let plan = scan - .plan_snapshot( - snapshot, - data_evolution_read_field_ids.as_ref(), - Some(&mut trace), - ) - .await?; - trace.planned_data_file_bytes = plan.planned_data_file_bytes(); - plan - } - None => Plan::new(Vec::new()), - }; - Ok((scan.finish_plan(plan).await?, trace)) - } - /// Keep pruning off masked columns and apply limits after row filtering. fn restricted_by(&self, grant: &Arc) -> Self { let rules = grant.rules(); @@ -1933,7 +1873,8 @@ impl<'a> PaimonTableScan<'a> { .iter() .map(|m| fields[m.column].name()) .collect(); - // By name: the bucket predicate is indexed by bucket-key position. + // Bucket selectors retain partition and bucket-key leaves. Any masked + // input must disable raw-value specialization of that selector. let touches_masked = |predicate: &Predicate| { super::query_auth::leaf_names(std::slice::from_ref(predicate)) .iter() @@ -1942,8 +1883,15 @@ impl<'a> PaimonTableScan<'a> { let mut scan = self.clone(); scan.query_auth_grant = Some(Arc::clone(grant)); scan.data_predicates.retain(|p| !touches_masked(p)); - if scan.bucket_predicate.as_ref().is_some_and(touches_masked) { - scan.bucket_predicate = None; + if let Some(predicate) = scan.bucket_predicate.take() { + let safe = predicate + .split_and() + .into_iter() + .filter(|predicate| !touches_masked(predicate)) + .collect::>(); + if !safe.is_empty() { + scan.bucket_predicate = Some(Predicate::and(safe)); + } } let dropped = scan.data_predicates.len() != self.data_predicates.len(); if !rules.filters.is_empty() || dropped { @@ -2020,7 +1968,7 @@ impl<'a> PaimonTableScan<'a> { &self, start: i64, end: i64, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { let Some((start, end)) = super::incremental_scan::timestamp_snapshot_range( &self.table.snapshot_manager(), @@ -2038,7 +1986,7 @@ impl<'a> PaimonTableScan<'a> { start, end, ) - .plan_combined_with_trace(trace) + .plan_combined_inner(scan_stats) .await } @@ -2303,18 +2251,20 @@ impl<'a> PaimonTableScan<'a> { &self, snapshot: &Snapshot, ) -> crate::Result> { - self.plan_manifest_entries_with_trace(ManifestListSource::Snapshot(snapshot), None, None) - .await + self.read_filtered_manifest_entries( + ManifestListSource::Snapshot(snapshot), + None, + &mut ScanStats::default(), + ) + .await } - async fn plan_manifest_entries_with_trace( + async fn read_filtered_manifest_entries( &self, source: ManifestListSource<'_>, row_range_index: Option<&RowRangeIndex>, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result> { - let file_io = self.table.file_io(); - let table_path = self.table.location(); let core_options = CoreOptions::new(self.table.schema().options()); let data_evolution_enabled = core_options.data_evolution_enabled(); @@ -2341,33 +2291,15 @@ impl<'a> PaimonTableScan<'a> { let key_fields = self.table.schema().trimmed_primary_keys(); let pushdown_key_predicates = self.key_stats_predicates(&pushdown_data_predicates); - let bucket_key_fields: Vec = if self.bucket_predicate.is_none() { - Vec::new() - } else { - let bucket_keys = core_options.bucket_key().unwrap_or_else(|| { - if has_primary_keys { - self.table.schema().trimmed_primary_keys() - } else { - Vec::new() - } - }); - bucket_keys - .iter() - .filter_map(|key| { - self.table - .schema() - .fields() - .iter() - .find(|f| f.name() == key) - .cloned() - }) - .collect::>() - }; - let bucket_function_type = core_options.bucket_function_type()?; + let bucket_selector = self + .bucket_predicate + .as_ref() + .map(|predicate| BucketSelector::new(&self.table, predicate)) + .transpose()? + .flatten(); let entries = read_all_manifest_entries( - file_io, - table_path, + &self.table, source, skip_level_zero, self.scan_all_files, @@ -2379,13 +2311,11 @@ impl<'a> PaimonTableScan<'a> { self.table.schema().id(), self.table.schema().fields(), &key_fields, - self.bucket_predicate.as_ref(), - &bucket_key_fields, - bucket_function_type, + bucket_selector.as_ref(), row_range_index, core_options.manifest_sidecar_enabled(), core_options.scan_manifest_parallelism()?, - trace, + scan_stats, ) .await?; Ok(entries) @@ -2614,7 +2544,11 @@ impl<'a> PaimonTableScan<'a> { /// /// Reuses the same split-building path as a full snapshot plan, but only /// reads the delta manifest list and keeps ADD entries. - pub(crate) async fn plan_snapshot_delta(&self, snapshot: &Snapshot) -> crate::Result { + pub(crate) async fn plan_snapshot_delta( + &self, + snapshot: &Snapshot, + scan_stats: &mut ScanStats, + ) -> crate::Result { self.validate_read_options()?; let data_evolution_read_field_ids = self.projected_read_field_ids()?; let mut scan = self.clone(); @@ -2623,7 +2557,7 @@ impl<'a> PaimonTableScan<'a> { snapshot, snapshot.delta_manifest_list(), data_evolution_read_field_ids.as_ref(), - None, + scan_stats, ) .await } @@ -2635,7 +2569,7 @@ impl<'a> PaimonTableScan<'a> { &self, end_snapshot: &Snapshot, manifest_lists: Vec, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { self.validate_read_options()?; let data_evolution_read_field_ids = self.projected_read_field_ids()?; @@ -2645,7 +2579,7 @@ impl<'a> PaimonTableScan<'a> { end_snapshot, ManifestListSource::IncrementalLists(manifest_lists), data_evolution_read_field_ids.as_ref(), - trace, + scan_stats, ) .await } @@ -2658,7 +2592,7 @@ impl<'a> PaimonTableScan<'a> { pub(crate) async fn plan_snapshot_changelog( &self, snapshot: &Snapshot, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { self.validate_read_options()?; let Some(list_name) = snapshot.changelog_manifest_list() else { @@ -2671,7 +2605,7 @@ impl<'a> PaimonTableScan<'a> { snapshot, list_name, data_evolution_read_field_ids.as_ref(), - trace, + scan_stats, ) .await } @@ -2681,13 +2615,13 @@ impl<'a> PaimonTableScan<'a> { snapshot: &Snapshot, manifest_list_name: &str, data_evolution_read_field_ids: Option<&HashSet>, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { self.plan_snapshot_from_lists( snapshot, ManifestListSource::IncrementalList(manifest_list_name), data_evolution_read_field_ids, - trace, + scan_stats, ) .await } @@ -2727,8 +2661,22 @@ impl<'a> PaimonTableScan<'a> { full_state_scan .validate_diff_bucket_layout(before, after) .await?; - let before_entries = full_state_scan.plan_manifest_entries(before).await?; - let after_entries = full_state_scan.plan_manifest_entries(after).await?; + let mut before_stats = ScanStats::default(); + let mut after_stats = ScanStats::default(); + let before_entries = full_state_scan + .read_filtered_manifest_entries( + ManifestListSource::Snapshot(before), + None, + &mut before_stats, + ) + .await?; + let after_entries = full_state_scan + .read_filtered_manifest_entries( + ManifestListSource::Snapshot(after), + None, + &mut after_stats, + ) + .await?; // Each side is a complete snapshot state. A DV index belongs to that // snapshot, so resolve the two index manifests independently before // building splits. Reusing the endpoint's DVs for the older state can @@ -2746,7 +2694,7 @@ impl<'a> PaimonTableScan<'a> { before_index_entries, None, &[], - None, + &mut before_stats, ) .await?; let after_plan = full_state_scan @@ -2757,10 +2705,13 @@ impl<'a> PaimonTableScan<'a> { after_index_entries, None, &[], - None, + &mut after_stats, ) .await?; - Ok((before_plan, after_plan)) + Ok(( + before_plan.with_scan_stats(before_stats), + after_plan.with_scan_stats(after_stats), + )) } async fn validate_diff_bucket_layout( @@ -2790,13 +2741,13 @@ impl<'a> PaimonTableScan<'a> { &self, snapshot: Snapshot, data_evolution_read_field_ids: Option<&HashSet>, - trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { self.plan_snapshot_from_lists( &snapshot, ManifestListSource::Snapshot(&snapshot), data_evolution_read_field_ids, - trace, + scan_stats, ) .await } @@ -2806,12 +2757,11 @@ impl<'a> PaimonTableScan<'a> { snapshot: &Snapshot, source: ManifestListSource<'_>, data_evolution_read_field_ids: Option<&HashSet>, - mut trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { + scan_stats.limit = self.limit; if matches!(self.limit, Some(0)) { - if let Some(trace) = trace { - trace.record_final_plan_with_limit(0, 0, 0, 0, true); - } + scan_stats.record_split_planning(0, true); return Ok(Plan::new(Vec::new()).with_snapshot_id(snapshot.id())); } let core_options = CoreOptions::new(self.table.schema().options()); @@ -2830,9 +2780,7 @@ impl<'a> PaimonTableScan<'a> { .manifest_row_ranges(snapshot, index_entries.as_deref(), global_index_settings) .await?; if manifest_row_ranges.as_ref().is_some_and(Vec::is_empty) { - if let Some(trace) = trace { - trace.record_final_plan(0, 0, 0); - } + scan_stats.record_split_planning(0, false); return Ok(Plan::new(Vec::new()).with_snapshot_id(snapshot.id())); } // Positional scans count all candidate row IDs. Pruning a preceding @@ -2844,11 +2792,7 @@ impl<'a> PaimonTableScan<'a> { None }; let entries = self - .plan_manifest_entries_with_trace( - source, - row_range_index.as_ref(), - trace.as_deref_mut(), - ) + .read_filtered_manifest_entries(source, row_range_index.as_ref(), scan_stats) .await?; let effective_row_ranges = self .effective_row_ranges( @@ -2866,7 +2810,7 @@ impl<'a> PaimonTableScan<'a> { index_entries, effective_row_ranges, &pk_sorted_index_definitions, - trace, + scan_stats, ) .await } @@ -2880,7 +2824,7 @@ impl<'a> PaimonTableScan<'a> { index_entries: Option>, effective_row_ranges: Option>, pk_sorted_index_definitions: &[super::pk_sorted_index_scan::Definition], - mut trace: Option<&mut ScanTrace>, + scan_stats: &mut ScanStats, ) -> crate::Result { let table_path = self.table.location(); let table_schema_id = self.table.schema().id(); @@ -2911,16 +2855,12 @@ impl<'a> PaimonTableScan<'a> { effective_row_ranges }; if effective_row_ranges.as_ref().is_some_and(Vec::is_empty) { - if let Some(trace) = trace { - trace.record_final_plan(0, 0, 0); - } + scan_stats.record_split_planning(0, false); return Ok(Plan::new(Vec::new()).with_snapshot_id(snapshot.id())); } if entries.is_empty() { - if let Some(trace) = trace { - trace.record_final_plan(0, 0, 0); - } + scan_stats.record_split_planning(0, false); return Ok(Plan::new(Vec::new()).with_snapshot_id(snapshot.id())); } @@ -2936,9 +2876,7 @@ impl<'a> PaimonTableScan<'a> { .iter() .any(|e| e.file().schema_id != current_schema_id); if !has_cross_schema { - if let Some(trace) = trace.as_deref_mut() { - trace.manifest_entries_after_cross_schema_stats = entries.len(); - } + scan_stats.manifest_entries_after_cross_schema_stats = entries.len(); entries } else { let before = entries.len(); @@ -2959,28 +2897,22 @@ impl<'a> PaimonTableScan<'a> { kept.push(entry); } } - if let Some(trace) = trace.as_deref_mut() { - trace.manifest_entries_pruned_by_cross_schema_stats += before - kept.len(); - trace.manifest_entries_after_cross_schema_stats = kept.len(); - } + scan_stats.manifest_entries_pruned_by_cross_schema_stats += before - kept.len(); + scan_stats.manifest_entries_after_cross_schema_stats = kept.len(); kept } }; if entries.is_empty() { - if let Some(trace) = trace { - trace.record_final_plan(0, 0, 0); - } + scan_stats.record_split_planning(0, false); return Ok(Plan::new(Vec::new()).with_snapshot_id(snapshot.id())); - } else if let Some(trace) = trace.as_deref_mut() { - if trace.manifest_entries_after_cross_schema_stats == 0 { - trace.manifest_entries_after_cross_schema_stats = entries.len(); + } else { + if scan_stats.manifest_entries_after_cross_schema_stats == 0 { + scan_stats.manifest_entries_after_cross_schema_stats = entries.len(); } } if matches!(self.limit, Some(0)) { - if let Some(trace) = trace { - trace.record_final_plan_with_limit(0, 0, 0, 0, true); - } + scan_stats.record_split_planning(0, true); return Ok(Plan::new(Vec::new()).with_snapshot_id(snapshot.id())); } @@ -3072,10 +3004,9 @@ impl<'a> PaimonTableScan<'a> { } else if data_evolution_enabled { let (row_id_groups, groups_pruned_by_row_ranges) = data_evolution_row_range_groups(data_files, effective_row_ranges.as_deref())?; - if let Some(trace) = trace.as_deref_mut() { - trace.data_evolution_groups_before_stats += row_id_groups.len(); - trace.data_evolution_groups_pruned_by_row_ranges += groups_pruned_by_row_ranges; - } + scan_stats.data_evolution_groups_before_stats += row_id_groups.len(); + scan_stats.data_evolution_groups_pruned_by_row_ranges += + groups_pruned_by_row_ranges; // Filter groups by merged stats before splitting. let row_id_groups: Vec> = if self.data_predicates.is_empty() { @@ -3095,9 +3026,7 @@ impl<'a> PaimonTableScan<'a> { groups.push(group); } } - if let Some(trace) = trace.as_deref_mut() { - trace.data_evolution_groups_pruned_by_stats += before - groups.len(); - } + scan_stats.data_evolution_groups_pruned_by_stats += before - groups.len(); groups }; @@ -3310,17 +3239,7 @@ impl<'a> PaimonTableScan<'a> { } else { splits }; - let splits_before_limit = split_candidates_built; - if let Some(trace) = trace { - let final_files = splits.iter().map(|split| split.data_files().len()).sum(); - trace.record_final_plan_with_limit( - split_candidates_built, - splits_before_limit, - splits.len(), - final_files, - limit_early_stopped, - ); - } + scan_stats.record_split_planning(split_candidates_built, limit_early_stopped); Ok(Plan::new(splits).with_snapshot_id(snapshot_id)) } @@ -3355,6 +3274,7 @@ mod tests { data_file_matches_predicates_for_table, data_file_matches_predicates_with_key_stats, group_by_overlapping_row_id, }; + use crate::table::ScanStats; use crate::table::{CommitMessage, Table, TableCommit}; use crate::Error; use bytes::Bytes; @@ -3783,7 +3703,7 @@ mod tests { None, None, &[], - None, + &mut ScanStats::default(), ) .await .unwrap(); @@ -3919,7 +3839,7 @@ mod tests { } } - fn scan_trace_test_table(table_path: &str) -> Table { + fn scan_stats_test_table(table_path: &str) -> Table { let file_io = FileIOBuilder::new("memory").build().unwrap(); let schema = PaimonSchema::builder() .column("id", DataType::Int(IntType::new())) @@ -3928,15 +3848,15 @@ mod tests { let table_schema = TableSchema::new(0, &schema); Table::new( file_io, - Identifier::new("test_db", "scan_trace"), + Identifier::new("test_db", "scan_stats"), table_path.to_string(), table_schema, None, ) } - fn scan_trace_small_split_table(table_path: &str) -> Table { - scan_trace_test_table(table_path).copy_with_options(HashMap::from([ + fn scan_stats_small_split_table(table_path: &str) -> Table { + scan_stats_test_table(table_path).copy_with_options(HashMap::from([ ("source.split.target-size".to_string(), "1b".to_string()), ("source.split.open-file-cost".to_string(), "1b".to_string()), ])) @@ -3944,14 +3864,14 @@ mod tests { #[tokio::test] async fn test_plain_shard_uses_java_file_names_before_limit() { - let table = scan_trace_small_split_table("memory:/plain_file_shards"); - setup_scan_trace_dirs(&table).await; + let table = scan_stats_small_split_table("memory:/plain_file_shards"); + setup_scan_stats_dirs(&table).await; // Java String.hashCode fixtures: Aa and BB collide at 2112; // polygenelubricants hashes to Integer.MIN_VALUE. let names = ["Aa", "BB", "polygenelubricants", "😀", "a"]; let files = names .iter() - .map(|name| stats_trace_file(name, 0, 1)) + .map(|name| stats_test_file(name, 0, 1)) .collect(); TableCommit::new(table.clone(), "file-shards".into()) .commit(vec![CommitMessage::new( @@ -4017,7 +3937,7 @@ mod tests { ("merge-engine".into(), engine.into()), ("source.split.target-size".into(), "1b".into()), ])); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; let commit = TableCommit::new(table.clone(), "bucket-shards".into()); for version in 0..2 { let messages = (0..4) @@ -4039,15 +3959,16 @@ mod tests { for index in 0..6 { let mut builder = table.new_read_builder(); builder.with_limit(1); - let (plan, trace) = builder + let plan = builder .new_scan() .with_shard(index, 6) .unwrap() - .plan_with_trace() + .plan() .await .unwrap(); + let scan_stats = plan.stats().clone(); assert_eq!(plan.snapshot_id(), Some(2)); - assert_eq!(trace.final_splits, plan.splits().len()); + assert_eq!(scan_stats.final_splits, plan.splits().len()); if index < 4 { assert_eq!(plan.splits().len(), 1); let split = &plan.splits()[0]; @@ -4110,7 +4031,7 @@ mod tests { ("deletion-vectors.enabled".into(), dv.to_string()), ("source.split.target-size".into(), "1b".into()), ])); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; let files = ["Aa", "BB", "polygenelubricants", "😀", "a"] .iter() .map(|name| pk_stats_file(name, (1, 2), (1, 2))) @@ -4171,7 +4092,7 @@ mod tests { two_column_schema(0, "id", "name"), ) .copy_with_options(HashMap::from([("bucket".into(), "-1".into())])); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; let messages = vec![CommitMessage::new( BinaryRowBuilder::new(0).build_serialized(), 0, @@ -4205,7 +4126,7 @@ mod tests { } } - pub(super) async fn setup_scan_trace_dirs(table: &Table) { + pub(super) async fn setup_scan_stats_dirs(table: &Table) { table .file_io() .mkdirs(&format!("{}/snapshot/", table.location())) @@ -4218,7 +4139,7 @@ mod tests { .unwrap(); } - fn stats_trace_file(name: &str, min_id: i32, max_id: i32) -> DataFileMeta { + fn stats_test_file(name: &str, min_id: i32, max_id: i32) -> DataFileMeta { let mut file = test_data_file_meta( int_stats_row(Some(min_id)), int_stats_row(Some(max_id)), @@ -4579,7 +4500,7 @@ mod tests { "1b".to_string(), )])); let table = data_evolution_test_table("memory:/row_position_selection_group_order", schema); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; TableCommit::new(table.clone(), "row-position-order".to_string()) .commit(vec![CommitMessage::new( BinaryRowBuilder::new(0).build_serialized(), @@ -4619,7 +4540,7 @@ mod tests { "memory:/row_position_selection_before_stats", two_column_schema(0, "id", "name"), ); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; let files = [0, 3] .into_iter() .map(|start| { @@ -4647,16 +4568,17 @@ mod tests { let mut reader = table.new_read_builder(); reader.with_filter(predicate).with_limit(1); - let (empty, trace) = reader + let empty = reader .new_scan() .with_row_position_shard(0, 2) .unwrap() - .plan_with_trace() + .plan() .await .unwrap(); + let scan_stats = empty.stats().clone(); assert!(empty.splits().is_empty()); assert_eq!(empty.snapshot_id(), Some(1)); - assert_eq!(trace.data_evolution_groups_pruned_by_stats, 1); + assert_eq!(scan_stats.data_evolution_groups_pruned_by_stats, 1); let plan = reader .new_scan() @@ -4682,7 +4604,7 @@ mod tests { "memory:/row_position_selection_range_intersection", two_column_schema(0, "id", "name"), ); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; TableCommit::new(table.clone(), "row-position-ranges".to_string()) .commit(vec![CommitMessage::new( BinaryRowBuilder::new(0).build_serialized(), @@ -4794,7 +4716,7 @@ mod tests { async fn test_data_evolution_row_ranges_prune_normal_groups() { let table_path = "memory:/de_row_range_prune_normal_groups"; let table = data_evolution_test_table(table_path, two_column_schema(0, "id", "name")); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; TableCommit::new(table.clone(), "row-range-prune-test".to_string()) .commit(vec![CommitMessage::new( @@ -4812,7 +4734,8 @@ mod tests { let mut read_builder = table.new_read_builder(); read_builder.with_row_ranges(vec![RowRange::new(0, 0)]); - let (plan, trace) = read_builder.new_scan().plan_with_trace().await.unwrap(); + let plan = read_builder.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); let planned_files = plan .splits() .iter() @@ -4821,12 +4744,12 @@ mod tests { .collect::>(); assert_eq!(planned_files, vec!["a-new", "a-old"]); - assert_eq!(trace.manifest_entries_read, 4); - assert_eq!(trace.manifest_entries_pruned_by_row_ranges, 2); - assert_eq!(trace.manifest_entries_after_manifest_filters, 4); - assert_eq!(trace.manifest_entries_after_merge, 4); - assert_eq!(trace.data_evolution_groups_before_stats, 1); - assert_eq!(trace.data_evolution_groups_pruned_by_row_ranges, 0); + assert_eq!(scan_stats.manifest_entries_read, 4); + assert_eq!(scan_stats.manifest_entries_pruned_by_row_ranges, 2); + assert_eq!(scan_stats.manifest_entries_after_manifest_filters, 4); + assert_eq!(scan_stats.manifest_entries_after_merge, 4); + assert_eq!(scan_stats.data_evolution_groups_before_stats, 1); + assert_eq!(scan_stats.data_evolution_groups_pruned_by_row_ranges, 0); let snapshot = table .snapshot_manager() @@ -4932,6 +4855,87 @@ mod tests { assert_eq!(sparse_bytes.as_ref(), expected); } + #[tokio::test] + async fn test_bucket_sidecar_keeps_pending_files_when_snapshot_scan_includes_them() { + let schema = PaimonSchema::builder() + .column("id", DataType::Int(IntType::new())) + .primary_key(["id"]) + .option("bucket", "-2") + .option("bucket-key", "id") + .option("manifest.sidecar.enabled", "true") + .build() + .unwrap(); + let table = Table::new( + FileIOBuilder::new("memory").build().unwrap(), + Identifier::new("db", "pending"), + "memory:/pending_bucket_sidecar".into(), + TableSchema::new(0, &schema), + None, + ); + setup_scan_stats_dirs(&table).await; + let predicate = PredicateBuilder::new(table.schema().fields()) + .equal("id", Datum::Int(3)) + .unwrap(); + let target = *compute_target_buckets( + &predicate, + table.schema().fields(), + BucketFunctionType::Default, + 4, + ) + .unwrap() + .iter() + .next() + .unwrap(); + let mut messages = Vec::new(); + for bucket in [-2, (target + 1) % 4] { + let files = (0..1000) + .map(|index| { + let mut file = stats_test_file(&format!("{bucket}-{index:04}.parquet"), 0, 100); + file.key_stats = BinaryTableStats::empty(); + file + }) + .collect(); + let mut message = + CommitMessage::new(BinaryRowBuilder::new(0).build_serialized(), bucket, files); + message.total_buckets = (bucket >= 0).then_some(4); + messages.push(message); + } + TableCommit::new(table.clone(), "pending".into()) + .commit(messages) + .await + .unwrap(); + let mut reader = table.new_read_builder(); + reader.with_filter(predicate); + let scan = reader.new_scan().with_scan_all_files(); + let plan = scan.plan().await.unwrap(); + let scan_stats = plan.stats(); + assert_eq!( + plan.splits() + .iter() + .map(|split| split.data_files().len()) + .sum::(), + 1000 + ); + assert!(plan.splits().iter().all(|split| split.bucket() == -2)); + assert!(scan_stats.manifest_entries_read >= 1000); + assert!( + scan_stats.manifest_entries_read < 2000, + "the sidecar should still prune assigned buckets" + ); + let snapshot = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + let initial = reader + .new_scan() + .plan_selected_snapshot(snapshot, super::SnapshotScanMode::All) + .await + .unwrap(); + assert_eq!(initial.splits(), plan.splits()); + } + #[tokio::test] async fn test_manifest_sidecar_prunes_avro_blocks_before_entry_decode() { const FILE_COUNT: usize = 1_000; @@ -4941,7 +4945,7 @@ mod tests { "true".to_string(), )])); let table = data_evolution_test_table(table_path, schema); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; let files = (0..FILE_COUNT) .map(|row_id| { @@ -4965,7 +4969,8 @@ mod tests { let mut reader = table.new_read_builder(); reader.with_row_ranges(vec![RowRange::new(0, 0)]); - let (plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); let planned = plan .splits() .iter() @@ -4974,18 +4979,18 @@ mod tests { .collect::>(); assert_eq!(planned, vec!["data-0000.parquet"]); - assert!(trace.manifest_entries_read > 0); + assert!(scan_stats.manifest_entries_read > 0); assert!( - trace.manifest_entries_read < FILE_COUNT, + scan_stats.manifest_entries_read < FILE_COUNT, "the sidecar should avoid decoding non-overlapping Avro blocks" ); } #[tokio::test] - async fn test_row_range_trace_excludes_manifest_netting() { - let table_path = "memory:/de_row_range_trace_netting"; + async fn test_row_range_stats_excludes_manifest_netting() { + let table_path = "memory:/de_row_range_stats_netting"; let table = data_evolution_test_table(table_path, two_column_schema(0, "id", "name")); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; let deleted = make_evo_file_with_cols("deleted.parquet", 10, 1, 0, &["id"]); let retained = make_evo_file_with_cols("retained.parquet", 10, 2, 0, &["id"]); @@ -5009,16 +5014,17 @@ mod tests { let mut read_builder = table.new_read_builder(); read_builder.with_row_ranges(vec![RowRange::new(0, 0)]); - let (plan, trace) = read_builder.new_scan().plan_with_trace().await.unwrap(); + let plan = read_builder.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); assert_eq!( plan.splits()[0].data_files()[0].file_name, "retained.parquet" ); - assert_eq!(trace.manifest_entries_read, 4); - assert_eq!(trace.manifest_entries_after_manifest_filters, 4); - assert_eq!(trace.manifest_entries_after_merge, 2); - assert_eq!(trace.manifest_entries_pruned_by_row_ranges, 1); + assert_eq!(scan_stats.manifest_entries_read, 4); + assert_eq!(scan_stats.manifest_entries_after_manifest_filters, 4); + assert_eq!(scan_stats.manifest_entries_after_merge, 2); + assert_eq!(scan_stats.manifest_entries_pruned_by_row_ranges, 1); } #[tokio::test] @@ -5029,7 +5035,7 @@ mod tests { "true".to_string(), )])); let table = data_evolution_test_table(table_path, schema); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; let deleted = make_evo_file_with_cols("deleted.blob", 100, 1, 0, &["payload"]); let partition = BinaryRowBuilder::new(0).build_serialized(); @@ -5252,7 +5258,7 @@ mod tests { None, None, &[], - None, + &mut ScanStats::default(), ) .await .unwrap(); @@ -5330,7 +5336,7 @@ mod tests { None, None, &[], - None, + &mut ScanStats::default(), ) .await .unwrap(); @@ -5427,7 +5433,7 @@ mod tests { None, None, &[], - None, + &mut ScanStats::default(), ) .await .unwrap(); @@ -5744,19 +5750,19 @@ mod tests { } #[tokio::test] - async fn test_plan_with_trace_records_between_data_stats_pruning() { - let table_path = "memory:/test_plan_with_trace_records_between_data_stats_pruning"; - let table = scan_trace_test_table(table_path); - setup_scan_trace_dirs(&table).await; + async fn test_scan_stats_records_between_data_stats_pruning() { + let table_path = "memory:/test_scan_stats_records_between_data_stats_pruning"; + let table = scan_stats_test_table(table_path); + setup_scan_stats_dirs(&table).await; - TableCommit::new(table.clone(), "scan-trace-test".to_string()) + TableCommit::new(table.clone(), "scan-stats-test".to_string()) .commit(vec![CommitMessage::new( BinaryRowBuilder::new(0).build_serialized(), 0, vec![ - stats_trace_file("stats-1.parquet", 1, 2), - stats_trace_file("stats-2.parquet", 10, 20), - stats_trace_file("stats-3.parquet", 100, 101), + stats_test_file("stats-1.parquet", 1, 2), + stats_test_file("stats-2.parquet", 10, 20), + stats_test_file("stats-3.parquet", 100, 101), ], )]) .await @@ -5770,16 +5776,237 @@ mod tests { ]); let mut reader = table.new_read_builder(); reader.with_filter(between); - let (_plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let _plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = _plan.stats().clone(); assert_eq!( - trace.final_files, 1, - "BETWEEN should keep only the overlapping stats range: {trace:?}" + scan_stats.final_files, 1, + "BETWEEN should keep only the overlapping stats range: {scan_stats:?}" ); assert!( - trace.manifest_entries_pruned_by_data_stats >= 2, - "BETWEEN should prune files outside the min/max range: {trace:?}" + scan_stats.manifest_entries_pruned_by_data_stats >= 2, + "BETWEEN should prune files outside the min/max range: {scan_stats:?}" + ); + } + + #[tokio::test] + async fn test_plan_statistics_preserve_normal_planning_with_limit() { + let table = scan_stats_test_table("memory:/scan_statistics_limit"); + setup_scan_stats_dirs(&table).await; + let empty = table.copy_with_pinned_snapshot(None); + let statistics_plan = empty.new_read_builder().new_scan().plan().await.unwrap(); + let stats = statistics_plan.stats(); + assert_eq!(stats, &super::super::ScanStats::default()); + TableCommit::new(table.clone(), "statistics".into()) + .commit(vec![CommitMessage::new( + BinaryRowBuilder::new(0).build_serialized(), + 0, + vec![ + stats_test_file("a.parquet", 1, 2), + stats_test_file("b.parquet", 10, 20), + stats_test_file("c.parquet", 100, 101), + ], + )]) + .await + .unwrap(); + let predicate = PredicateBuilder::new(table.schema().fields()) + .less_or_equal("id", Datum::Int(20)) + .unwrap(); + for limit in [None, Some(0), Some(1)] { + let mut reader = table.new_read_builder(); + reader.with_filter(predicate.clone()); + if let Some(limit) = limit { + reader.with_limit(limit); + } + let scan = reader.new_scan(); + let plan = scan.plan().await.unwrap(); + let stats = plan.stats(); + assert_eq!( + stats.planned_data_file_bytes, + plan.planned_data_file_bytes() + ); + assert_eq!(plan.snapshot_id(), Some(1)); + assert_eq!( + stats.manifest_entries_read, + if limit == Some(0) { 0 } else { 3 } + ); + assert_eq!( + stats.final_files, + plan.splits() + .iter() + .map(|s| s.data_files().len()) + .sum::() + ); + if limit == Some(0) { + assert!(stats.limit_early_stopped); + assert_eq!(stats.split_candidates_built, 0); + } + } + let plan = empty.new_read_builder().new_scan().plan().await.unwrap(); + let stats = plan.stats(); + assert_eq!(plan.snapshot_id(), None); + assert!(plan.splits().is_empty()); + assert_eq!(stats, &super::super::ScanStats::default()); + } + + #[tokio::test] + async fn test_plan_statistics_record_cross_schema_manifest_netting() { + let old_table = scan_stats_test_table("memory:/cross_schema_scan_stats"); + setup_scan_stats_dirs(&old_table).await; + write_schema_file(&old_table, old_table.schema()).await; + let deleted = stats_test_file("deleted.parquet", 1, 2); + let partition = BinaryRowBuilder::new(0).build_serialized(); + TableCommit::new(old_table.clone(), "old".into()) + .commit(vec![CommitMessage::new( + partition.clone(), + 0, + vec![deleted.clone()], + )]) + .await + .unwrap(); + let current_schema = old_table + .schema() + .apply_changes(vec![SchemaChange::rename_column( + "id".into(), + "renamed".into(), + )]) + .unwrap(); + let table = Table::new( + old_table.file_io().clone(), + Identifier::new("test_db", "scan_stats"), + old_table.location().into(), + current_schema, + None, + ); + write_schema_file(&table, table.schema()).await; + let mut retained = stats_test_file("retained.parquet", 100, 101); + retained.schema_id = table.schema().id(); + let mut message = CommitMessage::new(partition, 0, vec![retained]); + message.deleted_files = vec![deleted]; + TableCommit::new(table.clone(), "new".into()) + .commit(vec![message]) + .await + .unwrap(); + for threshold in [0, 50, 200] { + let mut reader = table.new_read_builder(); + reader.with_filter( + PredicateBuilder::new(table.schema().fields()) + .greater_than("renamed", Datum::Int(threshold)) + .unwrap(), + ); + let scan = reader.new_scan(); + let plan = scan.plan().await.unwrap(); + let stats = plan.stats(); + assert_eq!(stats.manifest_entries_read, 3); + assert_eq!( + stats.manifest_entries_after_merge, + usize::from(threshold < 200) + ); + assert_eq!( + stats.manifest_entries_pruned_by_data_stats, + usize::from(threshold == 200) + ); + } + } + + #[tokio::test] + async fn test_plan_statistics_record_actual_bucket_then_partition_pruning() { + let schema = PaimonSchema::builder() + .column("pt", DataType::Int(IntType::new())) + .column("id", DataType::Int(IntType::new())) + .partition_keys(["pt"]) + .primary_key(["pt", "id"]) + .option("bucket", "4") + .build() + .unwrap(); + let table = Table::new( + FileIOBuilder::new("memory").build().unwrap(), + Identifier::new("db", "stats"), + "memory:/partition_bucket_stats".into(), + TableSchema::new(0, &schema), + None, ); + setup_scan_stats_dirs(&table).await; + let mut messages = Vec::new(); + for partition in [1, 2] { + for bucket in 0..4 { + let mut row = BinaryRowBuilder::new(1); + row.write_int(0, partition); + let file = stats_test_file(&format!("{partition}-{bucket}.parquet"), 0, 100); + messages.push(CommitMessage::new( + row.build_serialized(), + bucket, + vec![file], + )); + } + } + TableCommit::new(table.clone(), "stats".into()) + .commit(messages) + .await + .unwrap(); + let pb = PredicateBuilder::new(table.schema().fields()); + let mut reader = table.new_read_builder(); + reader.with_filter(Predicate::and(vec![ + pb.equal("pt", Datum::Int(1)).unwrap(), + pb.equal("id", Datum::Int(3)).unwrap(), + ])); + let scan = reader.new_scan(); + let plan = scan.plan().await.unwrap(); + let stats = plan.stats(); + assert_eq!(stats.manifest_entries_read, 8); + // The bucket selector fails open for a nonmatching partition; the + // partition filter rejects those four entries after bucket evaluation. + assert_eq!(stats.manifest_entries_pruned_by_bucket, 3); + assert_eq!(stats.manifest_entries_pruned_by_partition, 4); + assert_eq!(stats.manifest_entries_after_entry_pruning, 1); + let mut reader = table.new_read_builder(); + reader.with_filter(pb.equal("pt", Datum::Int(99)).unwrap()); + let plan = reader.new_scan().plan().await.unwrap(); + let stats = plan.stats(); + assert!(plan.splits().is_empty()); + assert_eq!(stats.manifest_files_after_partition_pruning, 0); + assert_eq!(stats.manifest_entries_read, 0); + } + + #[tokio::test] + async fn test_plan_statistics_record_data_evolution_projection_and_limit() { + let table = data_evolution_test_table( + "memory:/evolution_scan_stats", + two_column_schema(0, "id", "name"), + ); + setup_scan_stats_dirs(&table).await; + let files = [ + ("id-a.parquet", 0, "id"), + ("name-a.parquet", 0, "name"), + ("id-b.parquet", 100, "id"), + ("name-b.parquet", 100, "name"), + ] + .into_iter() + .map(|(name, start, column)| make_evo_file_with_cols(name, 10, 1, start, &[column])) + .collect(); + TableCommit::new(table.clone(), "stats".into()) + .commit(vec![CommitMessage::new( + BinaryRowBuilder::new(0).build_serialized(), + 0, + files, + )]) + .await + .unwrap(); + for limit in [None, Some(0), Some(1)] { + let mut reader = table.new_read_builder(); + reader.with_projection(&["id"]).unwrap(); + if let Some(limit) = limit { + reader.with_limit(limit); + } + let scan = reader.new_scan(); + let plan = scan.plan().await.unwrap(); + let stats = plan.stats(); + assert_eq!( + stats.manifest_entries_read, + if limit == Some(0) { 0 } else { 4 } + ); + assert_eq!(stats.final_files, if limit == Some(0) { 0 } else { 2 }); + } } pub(super) fn pk_stats_gate_table(table_path: &str) -> Table { @@ -5873,7 +6100,7 @@ mod tests { async fn test_pk_table_stats_pruning_ignores_non_key_conjuncts() { let table_path = "memory:/test_pk_stats_gate"; let table = pk_stats_gate_table(table_path); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; // Both files cover key id=1; the newer version's value (50) falls // outside the value predicate while the older one (150) matches. @@ -5899,12 +6126,17 @@ mod tests { let value_filter = pb.greater_than("value", Datum::Int(90)).unwrap(); let mut reader = table.new_read_builder(); reader.with_filter(value_filter); - let (plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); assert_eq!( - trace.manifest_entries_pruned_by_data_stats, 0, - "non-key conjuncts must not file-prune a PK table: {trace:?}" + scan_stats.manifest_entries_pruned_by_data_stats, 0, + "non-key conjuncts must not file-prune a PK table: {scan_stats:?}" ); - let planned_files: usize = plan.splits().iter().map(|s| s.data_files().len()).sum(); + let planned_files: usize = plan + .splits() + .iter() + .map(|s| s.data_files().len()) + .sum::(); assert_eq!( planned_files, 2, "both versions must reach the merge reader" @@ -5914,10 +6146,11 @@ mod tests { let key_filter = pb.equal("id", Datum::Int(9)).unwrap(); let mut reader = table.new_read_builder(); reader.with_filter(key_filter); - let (_plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let _plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = _plan.stats().clone(); assert!( - trace.manifest_entries_pruned_by_data_stats >= 2, - "key conjuncts must still prune PK-table files: {trace:?}" + scan_stats.manifest_entries_pruned_by_data_stats >= 2, + "key conjuncts must still prune PK-table files: {scan_stats:?}" ); } @@ -5935,7 +6168,7 @@ mod tests { "true".to_string(), ), ])); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; let mut old = pk_stats_file("old-version.parquet", (1, 5), (100, 200)); old.level = 1; @@ -5959,9 +6192,10 @@ mod tests { .unwrap(); let mut reader = table.new_read_builder(); reader.with_filter(value_filter); - let (plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); - assert_eq!(trace.manifest_entries_pruned_by_data_stats, 0); + assert_eq!(scan_stats.manifest_entries_pruned_by_data_stats, 0); assert_eq!( plan.splits() .iter() @@ -5997,7 +6231,7 @@ mod tests { TableSchema::new(0, &schema), None, ); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; // Compacted (level 1) files: first-row planning skips level 0, so the // fixture files must sit above it to be planned at all. Distinct key @@ -6025,12 +6259,17 @@ mod tests { let value_filter = pb.greater_than("value", Datum::Int(90)).unwrap(); let mut reader = table.new_read_builder(); reader.with_filter(value_filter); - let (plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); assert!( - trace.manifest_entries_pruned_by_data_stats >= 1, - "first-row tables must keep full-predicate stats pruning: {trace:?}" + scan_stats.manifest_entries_pruned_by_data_stats >= 1, + "first-row tables must keep full-predicate stats pruning: {scan_stats:?}" ); - let planned_files: usize = plan.splits().iter().map(|s| s.data_files().len()).sum(); + let planned_files: usize = plan + .splits() + .iter() + .map(|s| s.data_files().len()) + .sum::(); assert_eq!( planned_files, 1, "only the value-matching file should be planned on first-row" @@ -6038,63 +6277,59 @@ mod tests { } #[tokio::test] - async fn test_plan_with_trace_records_limit_early_stop_during_split_construction() { + async fn test_scan_stats_records_limit_early_stop_during_split_construction() { let table_path = - "memory:/test_plan_with_trace_records_limit_early_stop_during_split_construction"; - let table = scan_trace_small_split_table(table_path); - setup_scan_trace_dirs(&table).await; + "memory:/test_scan_stats_records_limit_early_stop_during_split_construction"; + let table = scan_stats_small_split_table(table_path); + setup_scan_stats_dirs(&table).await; - TableCommit::new(table.clone(), "scan-trace-limit-test".to_string()) + TableCommit::new(table.clone(), "scan-stats-limit-test".to_string()) .commit(vec![CommitMessage::new( BinaryRowBuilder::new(0).build_serialized(), 0, vec![ - stats_trace_file("limit-1.parquet", 1, 1), - stats_trace_file("limit-2.parquet", 2, 2), - stats_trace_file("limit-3.parquet", 3, 3), + stats_test_file("limit-1.parquet", 1, 1), + stats_test_file("limit-2.parquet", 2, 2), + stats_test_file("limit-3.parquet", 3, 3), ], )]) .await .unwrap(); - let (_full_plan, full_trace) = table - .new_read_builder() - .new_scan() - .plan_with_trace() - .await - .unwrap(); + let _full_plan = table.new_read_builder().new_scan().plan().await.unwrap(); + let full_stats = _full_plan.stats().clone(); let mut limited_reader = table.new_read_builder(); limited_reader.with_limit(2); - let (_limited_plan, limited_trace) = - limited_reader.new_scan().plan_with_trace().await.unwrap(); + let _limited_plan = limited_reader.new_scan().plan().await.unwrap(); + let limited_stats = _limited_plan.stats().clone(); assert_eq!( - full_trace.final_splits, 3, - "fixture should build three splits: {full_trace:?}" + full_stats.final_splits, 3, + "fixture should build three splits: {full_stats:?}" ); - assert!(!full_trace.limit_early_stopped); - assert_eq!(full_trace.split_candidates_built, full_trace.final_splits); - assert!(limited_trace.limit_early_stopped); + assert!(!full_stats.limit_early_stopped); + assert_eq!(full_stats.split_candidates_built, full_stats.final_splits); + assert!(limited_stats.limit_early_stopped); assert!( - limited_trace.split_candidates_built < full_trace.split_candidates_built, - "limited trace should show construction-time stop: full={full_trace:?}, limited={limited_trace:?}" + limited_stats.split_candidates_built < full_stats.split_candidates_built, + "limited statistics should show construction-time stop: full={full_stats:?}, limited={limited_stats:?}" ); - assert_eq!(limited_trace.final_splits, 1); + assert_eq!(limited_stats.final_splits, 1); } #[tokio::test] - async fn test_plan_with_trace_zero_limit_records_no_split_candidates() { - let table_path = "memory:/test_plan_with_trace_zero_limit_records_no_split_candidates"; - let table = scan_trace_small_split_table(table_path); - setup_scan_trace_dirs(&table).await; + async fn test_scan_stats_zero_limit_records_no_split_candidates() { + let table_path = "memory:/test_scan_stats_zero_limit_records_no_split_candidates"; + let table = scan_stats_small_split_table(table_path); + setup_scan_stats_dirs(&table).await; - TableCommit::new(table.clone(), "scan-trace-zero-limit-test".to_string()) + TableCommit::new(table.clone(), "scan-stats-zero-limit-test".to_string()) .commit(vec![CommitMessage::new( BinaryRowBuilder::new(0).build_serialized(), 0, vec![ - stats_trace_file("zero-1.parquet", 1, 1), - stats_trace_file("zero-2.parquet", 2, 2), + stats_test_file("zero-1.parquet", 1, 1), + stats_test_file("zero-2.parquet", 2, 2), ], )]) .await @@ -6102,12 +6337,13 @@ mod tests { let mut reader = table.new_read_builder(); reader.with_limit(0); - let (plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); assert!(plan.splits().is_empty()); - assert!(trace.limit_early_stopped); - assert_eq!(trace.split_candidates_built, 0); - assert_eq!(trace.final_splits, 0); + assert!(scan_stats.limit_early_stopped); + assert_eq!(scan_stats.split_candidates_built, 0); + assert_eq!(scan_stats.final_splits, 0); } #[test] @@ -6879,7 +7115,7 @@ mod tests { #[tokio::test] async fn test_pk_index_scan_uses_resolved_definitions_instead_of_option_presence() { let table = pk_stats_gate_table("memory:/pk_index_definition_gate"); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; TableCommit::new(table.clone(), "definition-gate".into()) .commit(vec![CommitMessage::new( BinaryRow::new(0).to_serialized_bytes(), @@ -6914,7 +7150,9 @@ mod tests { ])); let scan = PaimonTableScan::new(&configured, None, vec![predicate.clone()], None, None, None); - let plan = scan.plan_snapshot(snapshot.clone(), None, None).await; + let plan = scan + .plan_snapshot(snapshot.clone(), None, &mut ScanStats::default()) + .await; if uses_index { assert!(plan .unwrap_err() @@ -7188,17 +7426,10 @@ mod tests { }; let buckets = compute_target_buckets(&pred, &fields, BucketFunctionType::Default, 4); - assert!(buckets.is_some(), "IsNull should determine a target bucket"); - let buckets = buckets.unwrap(); - assert_eq!(buckets.len(), 1); - let bucket = *buckets.iter().next().unwrap(); - assert!((0..4).contains(&bucket)); - - // Verify it matches the expected bucket from a null BinaryRow - let mut builder = BinaryRowBuilder::new(1); - builder.set_null_at(0); - let expected = (builder.build().hash_code() % 4).abs(); - assert_eq!(bucket, expected); + assert!( + buckets.is_none(), + "Java BucketSelector only extracts Equal/In values" + ); } #[test] @@ -7227,13 +7458,9 @@ mod tests { let buckets = compute_target_buckets(&pred, &fields, BucketFunctionType::Default, 8); assert!( - buckets.is_some(), - "Composite key with IsNull should determine a target bucket" + buckets.is_none(), + "one bucket key has no Equal/In constraint" ); - let buckets = buckets.unwrap(); - assert_eq!(buckets.len(), 1); - let bucket = *buckets.iter().next().unwrap(); - assert!((0..8).contains(&bucket)); } #[tokio::test] @@ -7329,8 +7556,8 @@ mod tests { #[tokio::test] async fn test_plan_snapshot_metadata_distinguishes_uncommitted_and_empty_results() { - let table = scan_trace_small_split_table("memory:/plan_snapshot_metadata"); - setup_scan_trace_dirs(&table).await; + let table = scan_stats_small_split_table("memory:/plan_snapshot_metadata"); + setup_scan_stats_dirs(&table).await; let builder = table.new_read_builder(); let plan = builder.new_scan().plan().await.unwrap(); @@ -7341,7 +7568,7 @@ mod tests { .commit(vec![CommitMessage::new( BinaryRowBuilder::new(0).build_serialized(), 0, - vec![stats_trace_file("metadata.parquet", 1, 1)], + vec![stats_test_file("metadata.parquet", 1, 1)], )]) .await .unwrap(); @@ -7356,9 +7583,10 @@ mod tests { let empty = limited.new_scan().plan().await.unwrap(); assert!(empty.splits().is_empty()); assert_eq!(empty.snapshot_id(), Some(1)); - let (empty, trace) = limited.new_scan().plan_with_trace().await.unwrap(); + let empty = limited.new_scan().plan().await.unwrap(); + let scan_stats = empty.stats().clone(); assert!(empty.splits().is_empty()); - assert_eq!(empty.snapshot_id(), trace.snapshot_id); + assert_eq!(scan_stats.final_splits, 0); assert_eq!(empty.snapshot_id(), Some(1)); let snapshot = table @@ -7376,7 +7604,7 @@ mod tests { assert_eq!(delta.snapshot_id(), Some(1)); let changelog = builder .new_scan() - .plan_snapshot_changelog_with_trace(&snapshot, None) + .plan_snapshot_changelog(&snapshot) .await .unwrap(); assert!(changelog.splits().is_empty()); @@ -7386,16 +7614,16 @@ mod tests { #[tokio::test] async fn test_plan_rejects_invalid_scan_manifest_parallelism() { let table = - scan_trace_test_table("memory:/invalid_scan_manifest_parallelism").copy_with_options( + scan_stats_test_table("memory:/invalid_scan_manifest_parallelism").copy_with_options( HashMap::from([("scan.manifest.parallelism".to_string(), "0".to_string())]), ); - setup_scan_trace_dirs(&table).await; + setup_scan_stats_dirs(&table).await; TableCommit::new(table.clone(), "manifest-parallelism-test".to_string()) .commit(vec![CommitMessage::new( BinaryRowBuilder::new(0).build_serialized(), 0, - vec![stats_trace_file("parallelism.parquet", 1, 1)], + vec![stats_test_file("parallelism.parquet", 1, 1)], )]) .await .unwrap(); @@ -7412,14 +7640,14 @@ mod tests { #[tokio::test] async fn test_plan_snapshot_metadata_preserves_pruned_historical_snapshot() { - let table = scan_trace_small_split_table("memory:/plan_historical_snapshot_metadata"); - setup_scan_trace_dirs(&table).await; + let table = scan_stats_small_split_table("memory:/plan_historical_snapshot_metadata"); + setup_scan_stats_dirs(&table).await; for id in 1..=2 { TableCommit::new(table.clone(), "historical-metadata-test".to_string()) .commit(vec![CommitMessage::new( BinaryRowBuilder::new(0).build_serialized(), 0, - vec![stats_trace_file(&format!("metadata-{id}.parquet"), id, id)], + vec![stats_test_file(&format!("metadata-{id}.parquet"), id, id)], )]) .await .unwrap(); @@ -7436,9 +7664,10 @@ mod tests { let plan = builder.new_scan().plan().await.unwrap(); assert!(plan.splits().is_empty()); assert_eq!(plan.snapshot_id(), Some(1)); - let (plan, trace) = builder.new_scan().plan_with_trace().await.unwrap(); + let plan = builder.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); assert!(plan.splits().is_empty()); - assert_eq!(plan.snapshot_id(), trace.snapshot_id); + assert_eq!(scan_stats.final_files, 0); assert_eq!(plan.snapshot_id(), Some(1)); let mut ranges = table.new_read_builder(); diff --git a/crates/paimon/src/table/table_write.rs b/crates/paimon/src/table/table_write.rs index 23675fe1e..1c52c6153 100644 --- a/crates/paimon/src/table/table_write.rs +++ b/crates/paimon/src/table/table_write.rs @@ -3136,12 +3136,13 @@ pub(in crate::table) mod tests { .unwrap(); let mut reader = table.new_read_builder(); reader.with_filter(predicate); - let (_plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let _plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = _plan.stats().clone(); - assert_eq!(trace.final_files, 1, "scan trace: {trace:?}"); + assert_eq!(scan_stats.final_files, 1, "scan scan_stats: {scan_stats:?}"); assert!( - trace.manifest_entries_pruned_by_data_stats >= 1, - "scan trace: {trace:?}" + scan_stats.manifest_entries_pruned_by_data_stats >= 1, + "scan scan_stats: {scan_stats:?}" ); } @@ -3438,9 +3439,13 @@ pub(in crate::table) mod tests { .unwrap(); let mut reader = table.new_read_builder(); reader.with_filter(predicate); - let (_plan, trace) = reader.new_scan().plan_with_trace().await.unwrap(); + let _plan = reader.new_scan().plan().await.unwrap(); + let scan_stats = _plan.stats().clone(); - assert!(trace.final_files > 0, "scan trace: {trace:?}"); + assert!( + scan_stats.final_files > 0, + "scan scan_stats: {scan_stats:?}" + ); assert_eq!( normal_write_cols, Some(vec!["a".to_string(), "b".to_string()]) diff --git a/crates/paimon/src/table/time_travel.rs b/crates/paimon/src/table/time_travel.rs index e2604cef1..245a9595e 100644 --- a/crates/paimon/src/table/time_travel.rs +++ b/crates/paimon/src/table/time_travel.rs @@ -605,12 +605,6 @@ mod tests { let changed = pinned.copy_with_options(extra); assert!(changed.resolve_read_snapshot().await.is_err()); assert!(changed.new_read_builder().new_scan().plan().await.is_err()); - assert!(changed - .new_read_builder() - .new_scan() - .plan_with_trace() - .await - .is_err()); } // Changing the selector must not silently reuse the empty view or the // schema captured by that view. Explicitly re-resolve the pair. diff --git a/crates/paimon/tests/file_index_java_fixture_test.rs b/crates/paimon/tests/file_index_java_fixture_test.rs index 7268dd1e3..3e9c47b4c 100644 --- a/crates/paimon/tests/file_index_java_fixture_test.rs +++ b/crates/paimon/tests/file_index_java_fixture_test.rs @@ -229,17 +229,18 @@ async fn reads_java_written_file_indexes_and_prunes_data_files() { let mut planning_builder = table.new_read_builder(); planning_builder.with_filter(missing.clone()); - let (plan, trace) = planning_builder + let plan = planning_builder .new_scan() - .plan_with_trace() + .plan() .await .expect("plan missing-value query"); + let scan_stats = plan.stats().clone(); assert_eq!( - trace.manifest_entries_pruned_by_data_stats, 0, + scan_stats.manifest_entries_pruned_by_data_stats, 0, "{}", case.table ); - assert_eq!(trace.final_files, 1, "{}", case.table); + assert_eq!(scan_stats.final_files, 1, "{}", case.table); let file = &plan.splits()[0].data_files()[0]; assert_eq!(file.row_count, 4, "{}", case.table); assert_eq!( diff --git a/crates/paimon/tests/incremental_changelog_batch_test.rs b/crates/paimon/tests/incremental_changelog_batch_test.rs index da6522833..aa1fa1d83 100644 --- a/crates/paimon/tests/incremental_changelog_batch_test.rs +++ b/crates/paimon/tests/incremental_changelog_batch_test.rs @@ -408,8 +408,8 @@ async fn combined_changelog_sharding_keeps_residual_filters_before_reader_limit( } #[tokio::test] -async fn timestamp_changelog_trace_describes_one_packed_batch() { - let table = table("trace", "1", false, "1mb").await; +async fn timestamp_changelog_stats_describes_one_packed_batch() { + let table = table("scan_stats", "1", false, "1mb").await; for id in 1..=3 { write_batch(&table, &make_batch(vec![1], vec![id])).await; patch_snapshot(&table, i64::from(id), "timeMillis", (id * 100).into()).await; @@ -418,21 +418,16 @@ async fn timestamp_changelog_trace_describes_one_packed_batch() { ("scan.mode".into(), "incremental".into()), ("incremental-between-timestamp".into(), "0,300".into()), ])); - let (plan, trace) = table - .new_read_builder() - .new_scan() - .plan_with_trace() - .await - .unwrap(); + let plan = table.new_read_builder().new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); assert_eq!(plan.snapshot_id(), Some(3)); - assert_eq!(trace.snapshot_id, Some(3)); - assert_eq!(trace.base_manifest_files, 0); - assert_eq!(trace.delta_manifest_files, 3); - assert_eq!(trace.manifest_entries_read, 3); - assert_eq!(trace.final_splits, 1); - assert_eq!(trace.final_files, 3); + assert_eq!(scan_stats.base_manifest_files, 0); + assert_eq!(scan_stats.delta_manifest_files, 3); + assert_eq!(scan_stats.manifest_entries_read, 3); + assert_eq!(scan_stats.final_splits, 1); + assert_eq!(scan_stats.final_files, 3); assert_eq!( - trace.planned_data_file_bytes, + scan_stats.planned_data_file_bytes, plan.splits() .iter() .flat_map(|split| split.data_files()) diff --git a/crates/paimon/tests/incremental_timestamp_scan_test.rs b/crates/paimon/tests/incremental_timestamp_scan_test.rs index 3d3dac333..9f0da25d3 100644 --- a/crates/paimon/tests/incremental_timestamp_scan_test.rs +++ b/crates/paimon/tests/incremental_timestamp_scan_test.rs @@ -157,16 +157,12 @@ async fn timestamp_options_select_events_and_preserve_end_snapshot() { assert_eq!(actual_snapshot, snapshot, "{mode}: ({start},{end}]"); assert_eq!(rows, expected, "{mode}: ({start},{end}]"); } - let (plan, trace) = selected - .new_read_builder() - .new_scan() - .plan_with_trace() - .await - .unwrap(); - assert_eq!(trace.snapshot_id, snapshot); - assert_eq!(trace.final_splits, plan.splits().len()); + let plan = selected.new_read_builder().new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); + assert_eq!(plan.snapshot_id(), snapshot); + assert_eq!(scan_stats.final_splits, plan.splits().len()); assert_eq!( - trace.planned_data_file_bytes, + scan_stats.planned_data_file_bytes, plan.planned_data_file_bytes() ); } @@ -186,10 +182,11 @@ async fn timestamp_auto_uses_physical_changelog_and_preserves_retracts() { append(&table, 300, vec![3], vec![30]).await; let selected = window(&table, 100, 300); let builder = selected.new_read_builder(); - let (plan, trace) = builder.new_scan().plan_with_trace().await.unwrap(); + let plan = builder.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); assert_eq!(plan.snapshot_id(), Some(3)); assert!(plan.splits().iter().all(|split| split.snapshot_id() == 3)); - assert!(trace.manifest_entries_read >= 2); + assert!(scan_stats.manifest_entries_read >= 2); let batches: Vec = builder .new_read() .unwrap() diff --git a/crates/paimon/tests/rest_catalog_test.rs b/crates/paimon/tests/rest_catalog_test.rs index 557ec5b0d..d51532cd0 100644 --- a/crates/paimon/tests/rest_catalog_test.rs +++ b/crates/paimon/tests/rest_catalog_test.rs @@ -5049,31 +5049,21 @@ async fn test_query_auth_chunk_shuffle_with_partition_filters() { .new_scan() .with_chunk_shuffle(7, 2) .expect("partition-only filters must allow chunk shuffle"); - for traced in [false, true] { - let plan = if traced { - scan.plan_with_trace().await.unwrap().0 - } else { - scan.plan().await.unwrap() - }; - assert_eq!(plan.splits().len(), 2); - assert!(plan - .splits() - .iter() - .all(|split| split.partition().get_string(0).unwrap() == "eng")); - let batches: Vec = read - .new_read() - .unwrap() - .to_arrow(plan.splits()) - .unwrap() - .try_collect() - .await - .unwrap(); - assert_eq!( - ids(&batches), - expected, - "evolution={evolution}, traced={traced}" - ); - } + let plan = scan.plan().await.unwrap(); + assert_eq!(plan.splits().len(), 2); + assert!(plan + .splits() + .iter() + .all(|split| split.partition().get_string(0).unwrap() == "eng")); + let batches: Vec = read + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(ids(&batches), expected, "evolution={evolution}"); } let data = predicates.greater_than("id", Datum::Int(2)).unwrap(); @@ -5100,19 +5090,13 @@ async fn test_query_auth_chunk_shuffle_with_partition_filters() { .unwrap(), ); let scan = read.new_scan().with_chunk_shuffle(7, 2).unwrap(); - for traced in [false, true] { - let result = if traced { - scan.plan_with_trace().await.map(|(plan, _)| plan) - } else { - scan.plan().await - }; - let error = result.unwrap_err(); - assert!( - matches!(error, paimon::Error::Unsupported { ref message } - if message.contains("unmasked partition predicates")), - "{error:?}" - ); - } + let result = scan.plan().await; + let error = result.unwrap_err(); + assert!( + matches!(error, paimon::Error::Unsupported { ref message } + if message.contains("unmasked partition predicates")), + "{error:?}" + ); } } @@ -5230,8 +5214,9 @@ async fn test_query_auth_filters_masked_partition_values() { equals("region", "east"), ])) .with_limit(1); - let (plan, trace) = limited.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.limit, None); + let plan = limited.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); + assert_eq!(scan_stats.limit, None); assert!(plan .splits() .iter() @@ -5259,8 +5244,9 @@ async fn test_query_auth_filters_masked_partition_values() { .with_filter(equals("dept", "eng")); assert_eq!(ids(&read_all(&plain).await.unwrap()), vec![1, 3]); plain.with_limit(1); - let (plan, trace) = plain.new_scan().plan_with_trace().await.unwrap(); - assert_eq!(trace.limit, Some(1)); + let plan = plain.new_scan().plan().await.unwrap(); + let scan_stats = plan.stats().clone(); + assert_eq!(scan_stats.limit, Some(1)); assert!(!plan.splits().is_empty()); let rows = ids(&read_all(&plain).await.unwrap()); assert_eq!(rows.len(), 1);